fix: sogs messages being skipped and not handled

ts/data/data.ts

@@ -46,11 +46,6 @@ export type SwarmNode = Snode & {
   address: string;
 };
 
-export type ServerToken = {
-  serverUrl: string;
-  token: string;
-};
-
 export const hasSyncedInitialConfigurationItem = 'hasSyncedInitialConfigurationItem';
 export const lastAvatarUploadTimestamp = 'lastAvatarUploadTimestamp';
 export const hasLinkPreviewPopupBeenDisplayed = 'hasLinkPreviewPopupBeenDisplayed';

ts/data/opengroups.ts

@@ -34,10 +34,6 @@ export type OpenGroupV2Room = {
   lastInboxIdFetched?: number;
   lastOutboxIdFetched?: number;
 
-  /**
-   * This value represents the rowId of the last message deleted. Not the id of the last message ID
-   */
-  lastMessageDeletedServerID?: number;
   /**
    * This value is set with the current timestamp whenever we get new messages.
    */

ts/node/sql.ts

@@ -1735,8 +1735,8 @@ function updateToLokiSchemaVersion27(currentVersion: number, db: BetterSqlite3.D
     const allSessionV2ConvosIp = compact(
       getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(ipToRemove))
     );
-    const allSessionV2ConvosDns = getAllOpenGroupV2Conversations(db).filter(m =>
-      m?.id.includes(domainNameToUse)
+    const allSessionV2ConvosDns = compact(
+      getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(domainNameToUse))
     );
 
     const duplicatesConvosIpAndDns = allSessionV2ConvosIp.filter(ip => {
@@ -1861,9 +1861,21 @@ function updateToLokiSchemaVersion27(currentVersion: number, db: BetterSqlite3.D
     });
 
     rebuildFtsTable(db);
-    console.log('... done');
+
+    console.info(
+      'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table'
+    );
+    db.exec(
+      `UPDATE ${OPEN_GROUP_ROOMS_V2_TABLE} SET
+      json = json_remove(json, '$.lastMessageDeletedServerID', '$.lastMessageFetchedServerID', '$.token' );`
+    );
+    console.info(
+      'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table. done'
+    );
     writeLokiSchemaVersion(targetVersion, db);
+    console.log('... done');
   })();
+
   console.log(`updateToLokiSchemaVersion${targetVersion}: success!`);
 }
 
@@ -3912,6 +3924,7 @@ function cleanUpOldOpengroupsOnStart() {
     console.info('cleanUpOldOpengroups: v2Convos is empty');
     return;
   }
+  console.info(`Count of v2 opengroup convos to clean: ${v2ConvosIds.length}`);
 
   // For each opengroups, if it has more than 1000 messages, we remove all the messages older than 2 months.
   // So this does not limit the size of opengroup history to 1000 messages but to 2 months.
@@ -3957,6 +3970,10 @@ function cleanUpOldOpengroupsOnStart() {
           convoProps.unreadCount = unreadCount;
           saveConversation(convoProps);
         }
+      } else {
+        console.info(
+          `Not cleaning messages older than 6 months in public convo: ${convoId}. message count: ${messagesInConvoBefore}`
+        );
       }
     });
 

ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts

@@ -145,7 +145,7 @@ const handleSogsV3DeletedMessages = async (
 ) => {
   // FIXME those 2 `m.data === null` test should be removed when we add support for emoji-reacts
   const deletions = messages.filter(m => Boolean(m.deleted) || m.data === null);
-  const exceptDeletion = messages.filter(m => !m.deleted && !m.data === null);
+  const exceptDeletion = messages.filter(m => !(Boolean(m.deleted) || m.data === null));
   if (!deletions.length) {
     return messages;
   }
@@ -170,6 +170,7 @@ const handleSogsV3DeletedMessages = async (
   return exceptDeletion;
 };
 
+// tslint:disable-next-line: cyclomatic-complexity
 const handleMessagesResponseV4 = async (
   messages: Array<OpenGroupMessageV4>,
   serverUrl: string,
@@ -268,7 +269,6 @@ const handleMessagesResponseV4 = async (
       roomInfosRefreshed.maxMessageFetchedSeqNo = maxNewMessageSeqNo;
     }
     roomInfosRefreshed.lastFetchTimestamp = Date.now();
-
     await OpenGroupData.saveV2OpenGroupRoom(roomInfosRefreshed);
   } catch (e) {
     window?.log?.warn('handleNewMessages failed:', e);
@@ -455,43 +455,43 @@ export const handleBatchPollResults = async (
   await handleCapabilities(subrequestOptionsLookup, batchPollResults, serverUrl);
 
   if (batchPollResults && isArray(batchPollResults.body)) {
-    await Promise.all(
-      batchPollResults.body.map(async (subResponse: any, index: number) => {
-        // using subreqOptions as request type lookup,
-        //assumes batch subresponse order matches the subrequest order
-        const subrequestOption = subrequestOptionsLookup[index];
-        const responseType = subrequestOption.type;
-
-        switch (responseType) {
-          case 'capabilities':
-            // capabilities are handled in handleCapabilities and are skipped here just to avoid the default case below
-            break;
-          case 'messages':
-            // this will also include deleted messages explicitly with `data: null` & edited messages with a new data field & react changes with data not existing
-            return handleMessagesResponseV4(
-              subResponse.body,
-              serverUrl,
-              subrequestOption,
-              roomIdsStillPolled
-            );
-          case 'pollInfo':
-            await handlePollInfoResponse(
-              subResponse.code,
-              subResponse.body,
-              serverUrl,
-              roomIdsStillPolled
-            );
-            break;
-          case 'inbox':
-            await handleInboxOutboxMessages(subResponse.body, serverUrl, false);
-            break;
-          case 'outbox':
-            await handleInboxOutboxMessages(subResponse.body, serverUrl, true);
-            break;
-          default:
-            window.log.error('No matching subrequest response body for type: ', responseType);
-        }
-      })
-    );
+    for (let index = 0; index < batchPollResults.body.length; index++) {
+      const subResponse = batchPollResults.body[index] as any;
+      // using subreqOptions as request type lookup,
+      //assumes batch subresponse order matches the subrequest order
+      const subrequestOption = subrequestOptionsLookup[index];
+      const responseType = subrequestOption.type;
+
+      switch (responseType) {
+        case 'capabilities':
+          // capabilities are handled in handleCapabilities and are skipped here just to avoid the default case below
+          break;
+        case 'messages':
+          // this will also include deleted messages explicitly with `data: null` & edited messages with a new data field & react changes with data not existing
+          await handleMessagesResponseV4(
+            subResponse.body,
+            serverUrl,
+            subrequestOption,
+            roomIdsStillPolled
+          );
+          break;
+        case 'pollInfo':
+          await handlePollInfoResponse(
+            subResponse.code,
+            subResponse.body,
+            serverUrl,
+            roomIdsStillPolled
+          );
+          break;
+        case 'inbox':
+          await handleInboxOutboxMessages(subResponse.body, serverUrl, false);
+          break;
+        case 'outbox':
+          await handleInboxOutboxMessages(subResponse.body, serverUrl, true);
+          break;
+        default:
+          window.log.error('No matching subrequest response body for type: ', responseType);
+      }
+    }
   }
 };