@ -73,6 +73,8 @@ module.exports = {
getMessagesByConversation ,
getFirstUnreadMessageIdInConversation ,
hasConversationOutgoingMessage ,
trimMessages ,
fillWithTestData ,
getUnprocessedCount ,
getAllUnprocessed ,
@ -836,6 +838,8 @@ const LOKI_SCHEMA_VERSIONS = [
updateToLokiSchemaVersion15 ,
updateToLokiSchemaVersion16 ,
updateToLokiSchemaVersion17 ,
updateToLokiSchemaVersion18 ,
updateToLokiSchemaVersion19 ,
] ;
function updateToLokiSchemaVersion1 ( currentVersion , db ) {
@ -1251,6 +1255,86 @@ function updateToLokiSchemaVersion17(currentVersion, db) {
console . log ( ` updateToLokiSchemaVersion ${ targetVersion } : success! ` ) ;
}
function updateToLokiSchemaVersion18 ( currentVersion , db ) {
const targetVersion = 18 ;
if ( currentVersion >= targetVersion ) {
return ;
}
console . log ( ` updateToLokiSchemaVersion ${ targetVersion } : starting... ` ) ;
// Dropping all pre-existing schema relating to message searching.
// Recreating the full text search and related triggers
db . transaction ( ( ) => {
db . exec ( `
DROP TRIGGER IF EXISTS messages _on _insert ;
DROP TRIGGER IF EXISTS messages _on _delete ;
DROP TRIGGER IF EXISTS messages _on _update ;
DROP TABLE IF EXISTS $ { MESSAGES _FTS _TABLE } ;
` );
writeLokiSchemaVersion ( targetVersion , db ) ;
} ) ( ) ;
db . transaction ( ( ) => {
db . exec ( `
-- Then we create our full - text search table and populate it
CREATE VIRTUAL TABLE $ { MESSAGES _FTS _TABLE }
USING fts5 ( id UNINDEXED , body ) ;
INSERT INTO $ { MESSAGES _FTS _TABLE } ( id , body )
SELECT id , body FROM $ { MESSAGES _TABLE } ;
-- Then we set up triggers to keep the full - text search table up to date
CREATE TRIGGER messages _on _insert AFTER INSERT ON $ { MESSAGES _TABLE } BEGIN
INSERT INTO $ { MESSAGES _FTS _TABLE } (
id ,
body
) VALUES (
new . id ,
new . body
) ;
END ;
CREATE TRIGGER messages _on _delete AFTER DELETE ON $ { MESSAGES _TABLE } BEGIN
DELETE FROM $ { MESSAGES _FTS _TABLE } WHERE id = old . id ;
END ;
CREATE TRIGGER messages _on _update AFTER UPDATE ON $ { MESSAGES _TABLE } BEGIN
DELETE FROM $ { MESSAGES _FTS _TABLE } WHERE id = old . id ;
INSERT INTO $ { MESSAGES _FTS _TABLE } (
id ,
body
) VALUES (
new . id ,
new . body
) ;
END ;
` );
writeLokiSchemaVersion ( targetVersion , db ) ;
} ) ( ) ;
console . log ( ` updateToLokiSchemaVersion ${ targetVersion } : success! ` ) ;
}
function updateToLokiSchemaVersion19 ( currentVersion , db ) {
const targetVersion = 19 ;
if ( currentVersion >= targetVersion ) {
return ;
}
console . log ( ` updateToLokiSchemaVersion ${ targetVersion } : starting... ` ) ;
db . transaction ( ( ) => {
db . exec ( `
DROP INDEX messages _schemaVersion ;
ALTER TABLE $ { MESSAGES _TABLE } DROP COLUMN schemaVersion ;
` );
// this is way to slow for now...
// db.exec(`
// UPDATE ${MESSAGES_TABLE} SET
// json = json_remove(json, '$.schemaVersion')
// `);
writeLokiSchemaVersion ( targetVersion , db ) ;
} ) ( ) ;
console . log ( ` updateToLokiSchemaVersion ${ targetVersion } : success! ` ) ;
}
function writeLokiSchemaVersion ( newVersion , db ) {
db . prepare (
` INSERT INTO loki_schema(
@ -1841,7 +1925,6 @@ function saveMessage(data) {
serverTimestamp ,
// eslint-disable-next-line camelcase
received _at ,
schemaVersion ,
sent ,
// eslint-disable-next-line camelcase
sent _at ,
@ -1876,7 +1959,6 @@ function saveMessage(data) {
hasFileAttachments ,
hasVisualMediaAttachments ,
received _at ,
schemaVersion ,
sent ,
sent _at ,
source ,
@ -1901,7 +1983,6 @@ function saveMessage(data) {
hasFileAttachments ,
hasVisualMediaAttachments ,
received _at ,
schemaVersion ,
sent ,
sent _at ,
source ,
@ -1922,7 +2003,6 @@ function saveMessage(data) {
$hasFileAttachments ,
$hasVisualMediaAttachments ,
$received _at ,
$schemaVersion ,
$sent ,
$sent _at ,
$source ,
@ -2220,6 +2300,135 @@ function getFirstUnreadMessageIdInConversation(conversationId) {
return rows [ 0 ] . id ;
}
/ * *
* Deletes all but the 10 , 000 last received messages .
* /
function trimMessages ( limit ) {
console . log ( limit ) ; // adding this for linting purposes.
// METHOD 1 Start - Seems to take to long and freeze
// const convoCount = globalInstance
// .prepare(
// `
// SELECT COUNT(*) FROM ${MESSAGES_TABLE}
// WHERE conversationId = $conversationId
// `
// )
// .all({
// conversationId,
// });
// if (convoCount < limit) {
// console.log(`Skipping conversation: ${conversationId}`);
// return;
// } else {
// console.count('convo surpassed limit');
// }
// globalInstance
// .prepare(
// `
// DELETE FROM ${MESSAGES_TABLE}
// WHERE conversationId = $conversationId
// AND id NOT IN (
// SELECT id FROM ${MESSAGES_TABLE}
// WHERE conversationId = $conversationId
// ORDER BY received_at DESC
// LIMIT $limit
// );
// `
// )
// .run({
// conversationId,
// limit,
// });
// METHOD 1 END
// METHOD 2 Start
// const messages = globalInstance
// .prepare(
// `
// SELECT id, conversationId FROM ${MESSAGES_TABLE}
// CREATE VIRTUAL TABLE IF NOT EXISTS temp_deletion
// id STRING PRIMARY KEY ASC
// `
// )
// .all();
// const idsToDelete = [];
// const convoCountLookup = {};
// for (let index = 0; index < messages.length; index + 1) {
// const { conversationId, id } = messages[index];
// console.log(`run ${index} - convoId: ${conversationId}, messageId: ${id}`);
// if (!convoCountLookup[conversationId]) {
// convoCountLookup[conversationId] = 1;
// } else {
// convoCountLookup[conversationId] + 1;
// if (convoCountLookup[conversationId] > limit) {
// idsToDelete.push(id);
// }
// }
// }
// // Ideally should be able to do WHERE id IN (x, y, z) with an array of IDs
// // the array might need to be chunked as well for performance
// const idSlice = [...idsToDelete].slice(0, 30);
// idSlice.forEach(() => {
// globalInstance
// .prepare(
// `
// DELETE FROM ${MESSAGES_TABLE}
// WHERE id = $idSlice
// `
// )
// .run({
// idSlice,
// });
// });
// Method 2 End
// Method 3 start - Audric's suggestion
// const largeConvos = globalInstance
// .prepare(
// `
// SELECT conversationId, count(id) FROM ${MESSAGES_TABLE}
// GROUP BY conversationId
// HAVING COUNT(id) > 1000
// `
// )
// .all();
// console.log({ largeConvos });
// // finding 1000th msg timestamp
// largeConvos.forEach(convo => {
// const convoId = convo.conversationId;
// console.log({ convoId });
// const lastMsg = globalInstance
// .prepare(
// `
// SELECT received_at, sent_at FROM ${MESSAGES_TABLE}
// WHERE conversationId = $convoId
// ORDER BY received_at DESC
// LIMIT 1
// OFFSET 999
// `
// )
// .all({
// convoId,
// });
// // use timestamp with lesserThan as conditional for deletion
// console.log({ lastMsg });
// const timestamp = lastMsg[0].received_at;
// if (timestamp) {
// console.log({ timestamp, convoId });
// globalInstance
// .prepare(
// `
// DELETE FROM ${MESSAGES_TABLE}
// WHERE conversationId = $convoId
// AND received_at < $timestamp
// `
// )
// .run({
// timestamp,
// convoId,
// });
// }
// });
}
function getMessagesBySentAt ( sentAt ) {
const rows = globalInstance
. prepare (
@ -2921,3 +3130,139 @@ function removeOneOpenGroupV1Message() {
return toRemoveCount - 1 ;
}
/ * *
* Only using this for development . Populate conversation and message tables .
* @ param { * } numConvosToAdd
* @ param { * } numMsgsToAdd
* /
function fillWithTestData ( numConvosToAdd , numMsgsToAdd ) {
const convoBeforeCount = globalInstance
. prepare ( ` SELECT count(*) from ${ CONVERSATIONS _TABLE } ; ` )
. get ( ) [ 'count(*)' ] ;
const lipsum =
// eslint:disable-next-line max-line-length
` Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis ac ornare lorem,
non suscipit purus . Lorem ipsum dolor sit amet , consectetur adipiscing elit .
Suspendisse cursus aliquet velit a dignissim . Integer at nisi sed velit consequat
dictum . Phasellus congue tellus ante . Ut rutrum hendrerit dapibus . Fusce
luctus , ante nec interdum molestie , purus urna volutpat turpis , eget mattis
lectus velit at velit . Praesent vel tellus turpis . Praesent eget purus at
nisl blandit pharetra . Cras dapibus sem vitae rutrum dapibus . Vivamus vitae mi
ante . Donec aliquam porta nibh , vel scelerisque orci condimentum sed .
Proin in mattis ipsum , ac euismod sem . Donec malesuada sem nisl , at
vehicula ante efficitur sed . Curabitur in sapien eros . Morbi tempor ante ut
metus scelerisque condimentum . Integer sit amet tempus nulla . Vivamus
imperdiet dui ac luctus vulputate . Sed a accumsan risus . Nulla facilisi .
Nulla mauris dui , luctus in sagittis at , sodales id mauris . Integer efficitur
viverra ex , ut dignissim eros tincidunt placerat . Sed facilisis gravida
mauris in luctus . Fusce dapibus , est vitae tincidunt eleifend , justo
odio porta dui , sed ultrices mi arcu vitae ante . Mauris ut libero
erat . Nam ut mi quis ante tincidunt facilisis sit amet id enim .
Vestibulum in molestie mi . In ac felis est . Vestibulum vel blandit ex . Morbi vitae
viverra augue . Ut turpis quam , cursus quis ex a , convallis
ullamcorper purus . Nam eget libero arcu . Integer fermentum enim nunc , non consequat urna
fermentum condimentum . Nulla vitae malesuada est . Donec imperdiet tortor interdum
malesuada feugiat . Integer pulvinar dui ex , eget tristique arcu mattis at . Nam eu neque
eget mauris varius suscipit . Quisque ac enim vitae mauris laoreet congue nec sed
justo . Curabitur fermentum quam eget est tincidunt , at faucibus lacus maximus . Donec
auctor enim dolor , faucibus egestas diam consectetur sed . Donec eget rutrum arcu , at
tempus mi . Fusce quis volutpat sapien . In aliquet fringilla purus . Ut eu nunc non
augue lacinia ultrices at eget tortor . Maecenas pulvinar odio sit amet purus
elementum , a vehicula lorem maximus . Pellentesque eu lorem magna . Vestibulum ut facilisis
lorem . Proin et enim cursus , vulputate neque sit amet , posuere enim . Praesent
faucibus tellus vel mi tincidunt , nec malesuada nibh malesuada . In laoreet sapien vitae
aliquet sollicitudin .
` ;
const msgBeforeCount = globalInstance . prepare ( ` SELECT count(*) from ${ MESSAGES _TABLE } ; ` ) . get ( ) [
'count(*)'
] ;
console . warn ( '==== fillWithTestData ====' ) ;
console . warn ( {
convoBeforeCount ,
msgBeforeCount ,
convoToAdd : numConvosToAdd ,
msgToAdd : numMsgsToAdd ,
} ) ;
const convosIdsAdded = [ ] ;
// eslint-disable-next-line no-plusplus
for ( let index = 0 ; index < numConvosToAdd ; index ++ ) {
const activeAt = Date . now ( ) - index ;
const id = Date . now ( ) - 1000 * index ;
const convoObjToAdd = {
active _at : activeAt ,
members : [ ] ,
profileName : ` ${ activeAt } ` ,
name : ` ${ activeAt } ` ,
id : ` 05 ${ id } ` ,
type : 'group' ,
} ;
convosIdsAdded . push ( id ) ;
try {
saveConversation ( convoObjToAdd ) ;
// eslint-disable-next-line no-empty
} catch ( e ) { }
}
console . warn ( 'convosIdsAdded' , convosIdsAdded ) ;
// eslint-disable-next-line no-plusplus
for ( let index = 0 ; index < numMsgsToAdd ; index ++ ) {
const activeAt = Date . now ( ) - index ;
const id = Date . now ( ) - 1000 * index ;
const lipsumStartIdx = Math . floor ( Math . random ( ) * lipsum . length ) ;
const lipsumLength = Math . floor ( Math . random ( ) * lipsum . length - lipsumStartIdx ) ;
const fakeBodyText = lipsum . substring ( lipsumStartIdx , lipsumStartIdx + lipsumLength ) ;
const convoId = convosIdsAdded [ Math . floor ( Math . random ( ) * convosIdsAdded . length ) ] ;
const msgObjToAdd = {
// body: `fake body ${activeAt}`,
body : ` fakeMsgIdx-spongebob- ${ index } ${ fakeBodyText } ${ activeAt } ` ,
conversationId : ` ${ convoId } ` ,
// eslint-disable-next-line camelcase
expires _at : 0 ,
hasAttachments : 0 ,
hasFileAttachments : 0 ,
hasVisualMediaAttachments : 0 ,
id : ` ${ id } ` ,
serverId : 0 ,
serverTimestamp : 0 ,
// eslint-disable-next-line camelcase
received _at : Date . now ( ) ,
sent : 0 ,
// eslint-disable-next-line camelcase
sent _at : Date . now ( ) ,
source : ` ${ convoId } ` ,
sourceDevice : 1 ,
type : '%' ,
unread : 1 ,
expireTimer : 0 ,
expirationStartTimestamp : 0 ,
} ;
if ( convoId % 10 === 0 ) {
console . info ( 'uyo , convoId ' , { index , convoId } ) ;
}
try {
saveMessage ( msgObjToAdd ) ;
// eslint-disable-next-line no-empty
} catch ( e ) {
console . warn ( e ) ;
}
}
const convoAfterCount = globalInstance
. prepare ( ` SELECT count(*) from ${ CONVERSATIONS _TABLE } ; ` )
. get ( ) [ 'count(*)' ] ;
const msgAfterCount = globalInstance . prepare ( ` SELECT count(*) from ${ MESSAGES _TABLE } ; ` ) . get ( ) [
'count(*)'
] ;
console . warn ( { convoAfterCount , msgAfterCount } ) ;
return convosIdsAdded ;
}
Audric Ackermann
2 years ago
committed by
GitHub