Merge pull request #2139 from warrickct/global-search-progress

Global Search and Database Trimming

_locales/en/messages.json

@@ -462,5 +462,6 @@
   "startedACall": "You called $name$",
   "answeredACall": "Call with $name$",
   "trimDatabase": "Trim Database",
-  "trimDatabaseDescription": "Reduces your message database size to your last 10,000 messages."
+  "trimDatabaseDescription": "Reduces your message database size to your last 10,000 messages.",
+  "trimDatabaseConfirmationBody": "Are you sure you want to delete your $deleteAmount$ oldest received messages?"
 }

app/sql.js

@@ -2303,29 +2303,130 @@ function getFirstUnreadMessageIdInConversation(conversationId) {
 /**
  * Deletes all but the 10,000 last received messages.
  */
-function trimMessages() {
+function trimMessages(limit) {
-  globalInstance
+  console.log(limit); // adding this for linting purposes.
-    .prepare(
+  // METHOD 1 Start - Seems to take to long and freeze
-      `
+  // const convoCount = globalInstance
-      DELETE FROM ${MESSAGES_TABLE}
+  //   .prepare(
-      WHERE id NOT IN (
+  //     `
-        SELECT id FROM ${MESSAGES_TABLE}
+  //       SELECT COUNT(*) FROM ${MESSAGES_TABLE}
-        ORDER BY received_at DESC
+  //       WHERE conversationId = $conversationId
-        LIMIT 10000
+  //     `
-      );
+  //   )
-    `
+  //   .all({
-    )
+  //     conversationId,
-    .run();
+  //   });
-
+  // if (convoCount < limit) {
-  const rows = globalInstance
+  //   console.log(`Skipping conversation: ${conversationId}`);
-    .prepare(
+  //   return;
-      `SELECT * FROM ${MESSAGES_TABLE}
+  // } else {
-     ORDER BY received_at DESC;`
+  //   console.count('convo surpassed limit');
-    )
+  // }
-    .all();
+  // globalInstance
-
+  //   .prepare(
-  return rows;
+  //     `
-  // return map(rows, row => jsonToObject(row.json));
+  //     DELETE FROM ${MESSAGES_TABLE}
+  //     WHERE conversationId = $conversationId
+  //     AND id NOT IN (
+  //       SELECT id FROM ${MESSAGES_TABLE}
+  //       WHERE conversationId = $conversationId
+  //       ORDER BY received_at DESC
+  //       LIMIT $limit
+  //     );
+  //   `
+  //   )
+  //   .run({
+  //     conversationId,
+  //     limit,
+  //   });
+  // METHOD 1 END
+  // METHOD 2 Start
+  // const messages = globalInstance
+  //   .prepare(
+  //     `
+  //       SELECT id, conversationId FROM ${MESSAGES_TABLE}
+  //       CREATE VIRTUAL TABLE IF NOT EXISTS temp_deletion
+  //       id STRING PRIMARY KEY ASC
+  //     `
+  //   )
+  //   .all();
+  // const idsToDelete = [];
+  // const convoCountLookup = {};
+  // for (let index = 0; index < messages.length; index + 1) {
+  //   const { conversationId, id } = messages[index];
+  //   console.log(`run ${index} - convoId: ${conversationId}, messageId: ${id}`);
+  //   if (!convoCountLookup[conversationId]) {
+  //     convoCountLookup[conversationId] = 1;
+  //   } else {
+  //     convoCountLookup[conversationId] + 1;
+  //     if (convoCountLookup[conversationId] > limit) {
+  //       idsToDelete.push(id);
+  //     }
+  //   }
+  // }
+  // // Ideally should be able to do WHERE id IN (x, y, z) with an array of IDs
+  // // the array might need to be chunked as well for performance
+  // const idSlice = [...idsToDelete].slice(0, 30);
+  // idSlice.forEach(() => {
+  //   globalInstance
+  //     .prepare(
+  //       `
+  //         DELETE FROM ${MESSAGES_TABLE}
+  //         WHERE id = $idSlice
+  //       `
+  //     )
+  //     .run({
+  //       idSlice,
+  //     });
+  // });
+  // Method 2 End
+  // Method 3 start - Audric's suggestion
+  // const largeConvos = globalInstance
+  //   .prepare(
+  //     `
+  //     SELECT conversationId, count(id) FROM ${MESSAGES_TABLE}
+  //     GROUP BY conversationId
+  //     HAVING COUNT(id) > 1000
+  //     `
+  //   )
+  //   .all();
+  // console.log({ largeConvos });
+  // // finding 1000th msg timestamp
+  // largeConvos.forEach(convo => {
+  //   const convoId = convo.conversationId;
+  //   console.log({ convoId });
+  //   const lastMsg = globalInstance
+  //     .prepare(
+  //       `
+  //         SELECT received_at, sent_at FROM ${MESSAGES_TABLE}
+  //         WHERE conversationId = $convoId
+  //         ORDER BY received_at DESC
+  //         LIMIT 1
+  //         OFFSET 999
+  //     `
+  //     )
+  //     .all({
+  //       convoId,
+  //     });
+  //   // use timestamp with lesserThan as conditional for deletion
+  //   console.log({ lastMsg });
+  //   const timestamp = lastMsg[0].received_at;
+  //   if (timestamp) {
+  //     console.log({ timestamp, convoId });
+  //     globalInstance
+  //       .prepare(
+  //         `
+  //           DELETE FROM ${MESSAGES_TABLE}
+  //           WHERE conversationId = $convoId
+  //           AND received_at < $timestamp
+  //         `
+  //       )
+  //       .run({
+  //         timestamp,
+  //         convoId,
+  //       });
+  //   }
+  // });
 }
 
 function getMessagesBySentAt(sentAt) {

stylesheets/_modules.scss

@@ -1455,7 +1455,7 @@
 .module-search-results {
   overflow-y: auto;
   max-height: 100%;
-  color: white;
+  color: var(--color-text);
 }
 
 .module-search-results__conversations-header {
@@ -1547,7 +1547,7 @@
   white-space: nowrap;
   text-overflow: ellipsis;
 
-  color: var(--color-text-subtle);
+  color: var(--color-text);
 }
 
 .module-message-search-result__header__timestamp {
@@ -1581,7 +1581,7 @@
 
   font-size: 13px;
 
-  color: $color-gray-60;
+  color: var(--color-text-subtle);
 
   max-height: 3.6em;
 

ts/components/basic/MessageBodyHighlight.tsx

@@ -0,0 +1,81 @@
+import React from 'react';
+import { RenderTextCallbackType } from '../../types/Util';
+import { SizeClassType } from '../../util/emoji';
+import { AddNewLines } from '../conversation/AddNewLines';
+import { Emojify } from '../conversation/Emojify';
+import { MessageBody } from '../conversation/message/message-content/MessageBody';
+
+const renderNewLines: RenderTextCallbackType = ({ text, key }) => (
+  <AddNewLines key={key} text={text} />
+);
+
+const renderEmoji = ({
+  text,
+  key,
+  sizeClass,
+  renderNonEmoji,
+}: {
+  text: string;
+  key: number;
+  sizeClass?: SizeClassType;
+  renderNonEmoji: RenderTextCallbackType;
+}) => <Emojify key={key} text={text} sizeClass={sizeClass} renderNonEmoji={renderNonEmoji} />;
+
+export const MessageBodyHighlight = (props: { text: string }) => {
+  const { text } = props;
+  const results: Array<JSX.Element> = [];
+  const FIND_BEGIN_END = /<<left>>(.+?)<<right>>/g;
+
+  let match = FIND_BEGIN_END.exec(text);
+  let last = 0;
+  let count = 1;
+
+  if (!match) {
+    return <MessageBody disableJumbomoji={true} disableLinks={true} text={text} />;
+  }
+
+  const sizeClass = '';
+
+  while (match) {
+    if (last < match.index) {
+      const beforeText = text.slice(last, match.index);
+      results.push(
+        renderEmoji({
+          text: beforeText,
+          sizeClass,
+          key: count++,
+          renderNonEmoji: renderNewLines,
+        })
+      );
+    }
+
+    const [, toHighlight] = match;
+    results.push(
+      <span className="module-message-body__highlight" key={count++}>
+        {renderEmoji({
+          text: toHighlight,
+          sizeClass,
+          key: count++,
+          renderNonEmoji: renderNewLines,
+        })}
+      </span>
+    );
+
+    // @ts-ignore
+    last = FIND_BEGIN_END.lastIndex;
+    match = FIND_BEGIN_END.exec(text);
+  }
+
+  if (last < text.length) {
+    results.push(
+      renderEmoji({
+        text: text.slice(last),
+        sizeClass,
+        key: count++,
+        renderNonEmoji: renderNewLines,
+      })
+    );
+  }
+
+  return <>{results}</>;
+};

ts/components/basic/MessageBodyHighlist.tsx

@@ -1,87 +0,0 @@
-import React from 'react';
-import { RenderTextCallbackType } from '../../types/Util';
-import { SizeClassType } from '../../util/emoji';
-import { AddNewLines } from '../conversation/AddNewLines';
-import { Emojify } from '../conversation/Emojify';
-import { MessageBody } from '../conversation/message/message-content/MessageBody';
-
-interface Props {
-  text: string;
-}
-
-const renderNewLines: RenderTextCallbackType = ({ text, key }) => (
-  <AddNewLines key={key} text={text} />
-);
-
-const renderEmoji = ({
-  text,
-  key,
-  sizeClass,
-  renderNonEmoji,
-}: {
-  text: string;
-  key: number;
-  sizeClass?: SizeClassType;
-  renderNonEmoji: RenderTextCallbackType;
-}) => <Emojify key={key} text={text} sizeClass={sizeClass} renderNonEmoji={renderNonEmoji} />;
-
-export class MessageBodyHighlight extends React.Component<Props> {
-  public render() {
-    const { text } = this.props;
-    const results: Array<any> = [];
-    const FIND_BEGIN_END = /<<left>>(.+?)<<right>>/g;
-
-    let match = FIND_BEGIN_END.exec(text);
-    let last = 0;
-    let count = 1;
-
-    if (!match) {
-      return <MessageBody disableJumbomoji={true} disableLinks={true} text={text} />;
-    }
-
-    const sizeClass = '';
-
-    while (match) {
-      if (last < match.index) {
-        const beforeText = text.slice(last, match.index);
-        results.push(
-          renderEmoji({
-            text: beforeText,
-            sizeClass,
-            key: count++,
-            renderNonEmoji: renderNewLines,
-          })
-        );
-      }
-
-      const [, toHighlight] = match;
-      results.push(
-        <span className="module-message-body__highlight" key={count++}>
-          {renderEmoji({
-            text: toHighlight,
-            sizeClass,
-            key: count++,
-            renderNonEmoji: renderNewLines,
-          })}
-        </span>
-      );
-
-      // @ts-ignore
-      last = FIND_BEGIN_END.lastIndex;
-      match = FIND_BEGIN_END.exec(text);
-    }
-
-    if (last < text.length) {
-      results.push(
-        renderEmoji({
-          text: text.slice(last),
-          sizeClass,
-          key: count++,
-          renderNonEmoji: renderNewLines,
-        })
-      );
-    }
-
-    return results;
-  }
-}

ts/components/search/MessageSearchResults.tsx

@@ -10,7 +10,7 @@ import {
 import { ContactName } from '../conversation/ContactName';
 import { Avatar, AvatarSize } from '../avatar/Avatar';
 import { Timestamp } from '../conversation/Timestamp';
-import { MessageBodyHighlight } from '../basic/MessageBodyHighlist';
+import { MessageBodyHighlight } from '../basic/MessageBodyHighlight';
 
 type PropsHousekeeping = {
   isSelected?: boolean;
@@ -29,7 +29,7 @@ export type PropsForSearchResults = {
   receivedAt?: number;
 };
 
-type Props = PropsForSearchResults & PropsHousekeeping;
+export type MessageResultProps = PropsForSearchResults & PropsHousekeeping;
 
 const FromName = (props: { source: string; destination: string }) => {
   const { source, destination } = props;
@@ -64,7 +64,6 @@ const From = (props: { source: string; destination: string }) => {
 
   const ourKey = getOurPubKeyStrFromCache();
 
-  // TODO: ww maybe add useConversationUsername hook within contact name
   if (destination !== ourKey) {
     return (
       <div className="module-message-search-result__header__from">
@@ -84,7 +83,7 @@ const AvatarItem = (props: { source: string }) => {
   return <Avatar size={AvatarSize.S} pubkey={source} />;
 };
 
-export const MessageSearchResult = (props: Props) => {
+export const MessageSearchResult = (props: MessageResultProps) => {
   const {
     isSelected,
     id,
@@ -96,6 +95,8 @@ export const MessageSearchResult = (props: Props) => {
     direction,
   } = props;
 
+  // Some messages miss a source or destination. Doing checks to see if the fields can be derived from other sources.
+  // E.g. if the source is missing but the message is outgoing, the source will be our pubkey
   const sourceOrDestinationDerivable =
     (destination && direction === MessageDirection.outgoing) ||
     !destination ||

ts/components/search/SearchResults.tsx

@@ -3,13 +3,12 @@ import {
   ConversationListItemProps,
   MemoConversationListItemWithDetails,
 } from '../leftpane/conversation-list-item/ConversationListItem';
-import { MessageSearchResult } from './MessageSearchResults';
+import { MessageResultProps, MessageSearchResult } from './MessageSearchResults';
 
 export type SearchResultsProps = {
   contacts: Array<ConversationListItemProps>;
   conversations: Array<ConversationListItemProps>;
-  // TODO: ww add proper typing
+  messages: Array<MessageResultProps>;
-  messages: Array<any>;
   hideMessagesHeader: boolean;
   searchTerm: string;
 };

ts/components/settings/section/CategoryAppearance.tsx

@@ -6,11 +6,7 @@ import useUpdate from 'react-use/lib/useUpdate';
 import {
   createOrUpdateItem,
   fillWithTestData,
-  fillWithTestData2,
-  // fillWithTestData2,
-  getMessageCount,
   hasLinkPreviewPopupBeenDisplayed,
-  trimMessages,
 } from '../../../data/data';
 import { ToastUtils } from '../../../session/utils';
 import { updateConfirmModal } from '../../../state/ducks/modalDialog';
@@ -139,12 +135,10 @@ export const SettingsCategoryAppearance = (props: { hasPassword: boolean | null
           buttonColor={SessionButtonColor.Primary}
           buttonText={window.i18n('translation')}
         />
-        <SessionSettingButtonItem
+        {/* <SessionSettingButtonItem
           title={window.i18n('trimDatabase')}
           description={window.i18n('trimDatabaseDescription')}
           onClick={async () => {
-            console.warn('trim the database to last 10k messages');
-
             const msgCount = await getMessageCount();
             const deleteAmount = Math.max(msgCount - 10000, 0);
 
@@ -156,13 +150,13 @@ export const SettingsCategoryAppearance = (props: { hasPassword: boolean | null
                 onClickClose: () => {
                   updateConfirmModal(null);
                 },
-                message: `Are you sure you want to delete your ${deleteAmount} oldest received messages?`,
+                message: window.i18n('trimDatabaseConfirmationBody', [`${deleteAmount}`]),
               })
             );
           }}
           buttonColor={SessionButtonColor.Primary}
           buttonText={window.i18n('trimDatabase')}
-        />
+        /> */}
         <SessionSettingButtonItem
           onClick={() => {
             ipcRenderer.send('show-debug-log');
@@ -172,14 +166,7 @@ export const SettingsCategoryAppearance = (props: { hasPassword: boolean | null
         />
         <SessionSettingButtonItem
           onClick={async () => {
-            await fillWithTestData(100, 2000000);
+            await fillWithTestData(100, 1000);
-          }}
-          buttonColor={SessionButtonColor.Primary}
-          buttonText={'Spam fill DB'}
-        />
-        <SessionSettingButtonItem
-          onClick={async () => {
-            await fillWithTestData2(100, 1000);
           }}
           buttonColor={SessionButtonColor.Primary}
           buttonText={'Spam fill DB using cached'}

ts/data/data.ts

@@ -823,8 +823,7 @@ export async function removeAllMessagesInConversation(conversationId: string): P
 }
 
 export async function trimMessages(): Promise<void> {
-  const count = await channels.trimMessages();
+  await channels.trimMessages(1000);
-  console.warn({ count });
   return;
 }
 
@@ -997,22 +996,7 @@ export async function removeOneOpenGroupV1Message(): Promise<number> {
  * @param numConvosToAdd Amount of fake conversations to generate
  * @param numMsgsToAdd Number of fake messages to generate
  */
-export async function fillWithTestData(
+export async function fillWithTestData(convs: number, msgs: number) {
-  numConvosToAdd: number,
-  numMsgsToAdd: number
-): Promise<void> {
-  if (!channels.fillWithTestData) {
-    return;
-  }
-  const ids = await channels.fillWithTestData(numConvosToAdd, numMsgsToAdd);
-  ids.map((id: string) => {
-    const convo = getConversationController().get(id);
-    const convoMsg = 'x';
-    convo.set('lastMessage', convoMsg);
-  });
-}
-
-export const fillWithTestData2 = async (convs: number, msgs: number) => {
   const newConvos = [];
   for (let convsAddedCount = 0; convsAddedCount < convs; convsAddedCount++) {
     const convoId = `${Date.now()} + ${convsAddedCount}`;
@@ -1038,4 +1022,4 @@ export const fillWithTestData2 = async (convs: number, msgs: number) => {
       direction: Math.random() > 0.5 ? 'outgoing' : 'incoming',
     });
   }
-};
+}

ts/state/ducks/search.ts

@@ -30,8 +30,7 @@ type SearchResultsPayloadType = {
   conversations: Array<string>;
   contacts: Array<string>;
 
-  // TODO: ww typing
+  messages?: Array<string>;
-  messages?: Array<any>;
 };
 
 type SearchResultsKickoffActionType = {

ts/types/LocalizerKeys.ts

@@ -464,4 +464,5 @@ export type LocalizerKeys =
   | 'editGroupName'
   | 'trimDatabase'
   | 'trimDatabaseDescription'
+  | 'trimDatabaseConfirmationBody'
   | 'reportIssue';