From 8feecb777b639006c1889ed5c768ecc613870319 Mon Sep 17 00:00:00 2001 From: audric Date: Wed, 12 Jan 2022 18:38:52 +1100 Subject: [PATCH] make sure to scale dowm preview in composition box --- js/background.js | 3 +- ts/components/SessionInboxView.tsx | 5 - .../conversation/SessionConversation.tsx | 165 +++++++++--------- .../composition/CompositionBox.tsx | 6 +- ts/session/utils/AttachmentsDownload.ts | 4 +- ts/types/MessageAttachment.ts | 16 +- ts/types/attachments/migrations.ts | 74 +++----- ts/util/attachmentsUtil.ts | 7 +- 8 files changed, 128 insertions(+), 152 deletions(-) diff --git a/js/background.js b/js/background.js index 8f5c8f32a..3c9abcf4c 100644 --- a/js/background.js +++ b/js/background.js @@ -144,7 +144,6 @@ shutdown: async () => { // Stop background processing window.libsession.Utils.AttachmentDownloads.stop(); - // Stop processing incoming messages // FIXME audric stop polling opengroupv2 and swarm nodes @@ -171,6 +170,8 @@ window.Events.setThemeSetting(newThemeSetting); try { + window.libsession.Utils.AttachmentDownloads.initAttachmentPaths(); + await Promise.all([ window.getConversationController().load(), BlockedNumberController.load(), diff --git a/ts/components/SessionInboxView.tsx b/ts/components/SessionInboxView.tsx index a4f282bcb..6a58f07f2 100644 --- a/ts/components/SessionInboxView.tsx +++ b/ts/components/SessionInboxView.tsx @@ -27,8 +27,6 @@ import { StateType } from '../state/reducer'; import { makeLookup } from '../util'; import { SessionMainPanel } from './SessionMainPanel'; import { createStore } from '../state/createStore'; -import { remote } from 'electron'; -import { initializeAttachmentLogic } from '../types/MessageAttachment'; // Workaround: A react component's required properties are filtering up through connect() // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/31363 @@ -77,9 +75,6 @@ export class SessionInboxView extends React.Component { } private setupLeftPane() { - const userDataPath = remote.app.getPath('userData'); - - initializeAttachmentLogic(userDataPath); // Here we set up a full redux store with initial state for our LeftPane Root const conversations = getConversationController() .getConversations() diff --git a/ts/components/conversation/SessionConversation.tsx b/ts/components/conversation/SessionConversation.tsx index fdea5e1fe..85a6c7917 100644 --- a/ts/components/conversation/SessionConversation.tsx +++ b/ts/components/conversation/SessionConversation.tsx @@ -33,14 +33,18 @@ import { SessionTheme } from '../../state/ducks/SessionTheme'; import { addStagedAttachmentsInConversation } from '../../state/ducks/stagedAttachments'; import { MIME } from '../../types'; import { AttachmentTypeWithPath } from '../../types/Attachment'; -import { AttachmentUtil, GoogleChrome } from '../../util'; +import { arrayBufferToObjectURL, AttachmentUtil, GoogleChrome } from '../../util'; import { SessionButtonColor } from '../basic/SessionButton'; import { MessageView } from '../MainViewController'; import { ConversationHeaderWithDetails } from './ConversationHeader'; import { MessageDetail } from './message/message-item/MessageDetail'; import { SessionRightPanelWithDetails } from './SessionRightPanel'; -import { autoOrientImage } from '../../types/attachments/migrations'; -import { makeVideoScreenshot } from '../../types/attachments/VisualAttachment'; +import { autoOrientJpegImage } from '../../types/attachments/migrations'; +import { + makeImageThumbnailBuffer, + makeVideoScreenshot, + THUMBNAIL_CONTENT_TYPE, +} from '../../types/attachments/VisualAttachment'; import { blobToArrayBuffer } from 'blob-util'; import { MAX_ATTACHMENT_FILESIZE_BYTES } from '../../session/constants'; // tslint:disable: jsx-curly-spacing @@ -338,80 +342,7 @@ export class SessionConversation extends React.Component { return; } - const renderVideoPreview = async () => { - const objectUrl = URL.createObjectURL(file); - try { - const type = 'image/png'; - - const thumbnail = await makeVideoScreenshot({ - objectUrl, - contentType: type, - }); - const data = await blobToArrayBuffer(thumbnail); - const url = window.Signal.Util.arrayBufferToObjectURL({ - data, - type, - }); - this.addAttachments([ - { - file, - size: file.size, - fileName, - contentType, - videoUrl: objectUrl, - url, - isVoiceMessage: false, - fileSize: null, - screenshot: null, - thumbnail: null, - }, - ]); - } catch (error) { - URL.revokeObjectURL(objectUrl); - } - }; - - const renderImagePreview = async () => { - if (!MIME.isJPEG(contentType)) { - const urlImage = URL.createObjectURL(file); - if (!urlImage) { - throw new Error('Failed to create object url for image!'); - } - this.addAttachments([ - { - file, - size: file.size, - fileName, - contentType, - url: urlImage, - isVoiceMessage: false, - fileSize: null, - screenshot: null, - thumbnail: null, - }, - ]); - return; - } - - const url = await autoOrientImage(file); - - this.addAttachments([ - { - file, - size: file.size, - fileName, - contentType, - url, - isVoiceMessage: false, - fileSize: null, - screenshot: null, - thumbnail: null, - }, - ]); - }; - let blob = null; - console.warn('typeof file: ', typeof file); try { blob = await AttachmentUtil.autoScale({ @@ -439,9 +370,11 @@ export class SessionConversation extends React.Component { // this is just for us, for the list of attachments we are sending // the files are scaled down under getFiles() - await renderImagePreview(); + const attachmentWithPreview = await renderImagePreview(contentType, file, fileName); + this.addAttachments([attachmentWithPreview]); } else if (GoogleChrome.isVideoTypeSupported(contentType)) { - await renderVideoPreview(); + const attachmentWithVideoPreview = await renderVideoPreview(contentType, file, fileName); + this.addAttachments([attachmentWithVideoPreview]); } else { this.addAttachments([ { @@ -542,3 +475,79 @@ export class SessionConversation extends React.Component { window.inboxStore?.dispatch(updateMentionsMembers(allMembers)); } } + +const renderVideoPreview = async (contentType: string, file: File, fileName: string) => { + const objectUrl = URL.createObjectURL(file); + try { + const type = THUMBNAIL_CONTENT_TYPE; + + const thumbnail = await makeVideoScreenshot({ + objectUrl, + contentType: type, + }); + const data = await blobToArrayBuffer(thumbnail); + const url = arrayBufferToObjectURL({ + data, + type, + }); + return { + file, + size: file.size, + fileName, + contentType, + videoUrl: objectUrl, + url, + isVoiceMessage: false, + fileSize: null, + screenshot: null, + thumbnail: null, + }; + } catch (error) { + URL.revokeObjectURL(objectUrl); + throw error; + } +}; + +const renderImagePreview = async (contentType: string, file: File, fileName: string) => { + if (!MIME.isJPEG(contentType)) { + const urlImage = URL.createObjectURL(file); + if (!urlImage) { + throw new Error('Failed to create object url for image!'); + } + return { + file, + size: file.size, + fileName, + contentType, + url: urlImage, + isVoiceMessage: false, + fileSize: null, + screenshot: null, + thumbnail: null, + }; + } + + // orient the image correctly based on the EXIF data, if needed + const orientedImageUrl = await autoOrientJpegImage(file); + + const thumbnailBuffer = await makeImageThumbnailBuffer({ + objectUrl: orientedImageUrl, + contentType, + }); + const url = arrayBufferToObjectURL({ + data: thumbnailBuffer, + type: THUMBNAIL_CONTENT_TYPE, + }); + + return { + file, + size: file.size, + fileName, + contentType, + url, + isVoiceMessage: false, + fileSize: null, + screenshot: null, + thumbnail: null, + }; +}; diff --git a/ts/components/conversation/composition/CompositionBox.tsx b/ts/components/conversation/composition/CompositionBox.tsx index 79abfa2fd..fc5d2e9f3 100644 --- a/ts/components/conversation/composition/CompositionBox.tsx +++ b/ts/components/conversation/composition/CompositionBox.tsx @@ -879,9 +879,7 @@ class CompositionBoxInner extends React.Component { return []; } // scale them down - const files = await Promise.all( - stagedAttachments.map(attachment => AttachmentUtil.getFileAndStoreLocally(attachment)) - ); + const files = await Promise.all(stagedAttachments.map(AttachmentUtil.getFileAndStoreLocally)); window.inboxStore?.dispatch( removeAllStagedAttachmentsInConversation({ conversationKey: this.props.selectedConversationKey, @@ -904,7 +902,7 @@ class CompositionBoxInner extends React.Component { const audioAttachment: StagedAttachmentType = { file: new File([], 'session-audio-message'), // this is just to emulate a file for the staged attachment type of that audio file contentType: MIME.AUDIO_MP3, - size: audioBlob.size, + size: savedAudioFile.size, fileSize: null, screenshot: null, fileName: 'session-audio-message', diff --git a/ts/session/utils/AttachmentsDownload.ts b/ts/session/utils/AttachmentsDownload.ts index 909af0f51..d7826ec3c 100644 --- a/ts/session/utils/AttachmentsDownload.ts +++ b/ts/session/utils/AttachmentsDownload.ts @@ -13,7 +13,7 @@ import { } from '../../../ts/data/data'; import { MessageModel } from '../../models/message'; import { downloadAttachment, downloadAttachmentOpenGroupV2 } from '../../receiver/attachments'; -import { processNewAttachment } from '../../types/MessageAttachment'; +import { initializeAttachmentLogic, processNewAttachment } from '../../types/MessageAttachment'; // this cause issues if we increment that value to > 1. const MAX_ATTACHMENT_JOB_PARALLELISM = 3; @@ -350,3 +350,5 @@ function _replaceAttachment(object: any, key: any, newAttachment: any, logPrefix // eslint-disable-next-line no-param-reassign object[key] = newAttachment; } + +export const initAttachmentPaths = initializeAttachmentLogic; diff --git a/ts/types/MessageAttachment.ts b/ts/types/MessageAttachment.ts index 7e9ed71e7..6f1ecd2db 100644 --- a/ts/types/MessageAttachment.ts +++ b/ts/types/MessageAttachment.ts @@ -1,3 +1,4 @@ +import { remote } from 'electron'; import { isArrayBuffer, isUndefined, omit } from 'lodash'; import { createAbsolutePathGetter, @@ -7,7 +8,7 @@ import { getPath, } from '../attachments/attachments'; import { - autoOrientJPEG, + autoOrientJPEGAttachment, captureDimensionsAndScreenshot, deleteData, loadData, @@ -16,7 +17,7 @@ import { // tslint:disable: prefer-object-spread // FIXME audric -// upgrade: exports._mapAttachments(autoOrientJPEG), +// upgrade: exports._mapAttachments(autoOrientJPEGAttachment), // upgrade: exports._mapAttachments(replaceUnicodeOrderOverrides), // upgrade: _mapAttachments(migrateDataToFileSystem), // upgrade: ._mapQuotedAttachments(migrateDataToFileSystem), @@ -86,7 +87,8 @@ let internalDeleteOnDisk: ((relativePath: string) => Promise) | undefined; let internalWriteNewAttachmentData: ((arrayBuffer: ArrayBuffer) => Promise) | undefined; // userDataPath must be app.getPath('userData'); -export function initializeAttachmentLogic(userDataPath: string) { +export function initializeAttachmentLogic() { + const userDataPath = remote.app.getPath('userData'); if (attachmentsPath) { throw new Error('attachmentsPath already initialized'); } @@ -108,7 +110,7 @@ export const getAttachmentPath = () => { return attachmentsPath; }; -export const loadAttachmentData = loadData(); +export const loadAttachmentData = loadData; export const loadPreviewData = async (preview: any) => { if (!preview || !preview.length) { @@ -160,11 +162,13 @@ export const processNewAttachment = async (attachment: { path?: string; isRaw?: boolean; }) => { - const rotatedData = await autoOrientJPEG(attachment); + // this operation might change the size (as we might print the content to a canvas and get the data back) + const rotatedData = await autoOrientJPEGAttachment(attachment); const rotatedAttachment = { ...attachment, contentType: rotatedData.contentType, data: rotatedData.data, + digest: attachment.digest as string | undefined, }; if (rotatedData.shouldDeleteDigest) { @@ -176,7 +180,7 @@ export const processNewAttachment = async (attachment: { const attachmentWithoutData = omit({ ...attachment, path: onDiskAttachmentPath }, 'data'); const finalAttachment = await captureDimensionsAndScreenshot(attachmentWithoutData); - return finalAttachment; + return { ...finalAttachment, size: rotatedAttachment.data.byteLength }; }; export const readAttachmentData = async (relativePath: string): Promise => { diff --git a/ts/types/attachments/migrations.ts b/ts/types/attachments/migrations.ts index 0ebc76bef..e215eec77 100644 --- a/ts/types/attachments/migrations.ts +++ b/ts/types/attachments/migrations.ts @@ -27,40 +27,15 @@ const DEFAULT_JPEG_QUALITY = 0.85; // // Documentation for `options` (`LoadImageOptions`): // https://github.com/blueimp/JavaScript-Load-Image/tree/v2.18.0#options -export const autoOrientImage = async ( - fileOrBlobOrURL: string | File | Blob, - options = {} +export const autoOrientJpegImage = async ( + fileOrBlobOrURL: string | File | Blob ): Promise => { - const optionsWithDefaults = { - type: 'image/jpeg', - quality: DEFAULT_JPEG_QUALITY, - ...options, - - canvas: true, - orientation: true, - maxHeight: 4096, // ATTACHMENT_DEFAULT_MAX_SIDE - maxWidth: 4096, - }; + const loadedImage = await loadImage(fileOrBlobOrURL, { orientation: true, canvas: true }); - return new Promise((resolve, reject) => { - loadImage( - fileOrBlobOrURL, - canvasOrError => { - if ((canvasOrError as any).type === 'error') { - const error = new Error('autoOrientImage: Failed to process image'); - (error as any).cause = canvasOrError; - reject(error); - return; - } - - const canvas = canvasOrError as HTMLCanvasElement; - const dataURL = canvas.toDataURL(optionsWithDefaults.type, optionsWithDefaults.quality); - - resolve(dataURL); - }, - optionsWithDefaults - ); - }); + const canvas = loadedImage.image as HTMLCanvasElement; + const dataURL = canvas.toDataURL(MIME.IMAGE_JPEG, DEFAULT_JPEG_QUALITY); + + return dataURL; }; // Returns true if `rawAttachment` is a valid attachment based on our current schema. @@ -87,7 +62,7 @@ const INVALID_CHARACTERS_PATTERN = new RegExp( // Upgrade steps // NOTE: This step strips all EXIF metadata from JPEG images as // part of re-encoding the image: -export const autoOrientJPEG = async (attachment: { +export const autoOrientJPEGAttachment = async (attachment: { contentType: string; data: ArrayBuffer; }): Promise<{ contentType: string; data: ArrayBuffer; shouldDeleteDigest: boolean }> => { @@ -101,7 +76,7 @@ export const autoOrientJPEG = async (attachment: { } const dataBlob = arrayBufferToBlob(attachment.data, attachment.contentType); - const newDataBlob = dataURLToBlob(await autoOrientImage(dataBlob)); + const newDataBlob = dataURLToBlob(await autoOrientJpegImage(dataBlob)); const newDataArrayBuffer = await blobToArrayBuffer(newDataBlob); // IMPORTANT: We overwrite the existing `data` `ArrayBuffer` losing the original @@ -172,28 +147,23 @@ export const _replaceUnicodeOrderOverridesSync = (attachment: any) => { export const hasData = (attachment: any) => attachment.data instanceof ArrayBuffer || ArrayBuffer.isView(attachment.data); -// loadData :: (RelativePath -> IO (Promise ArrayBuffer)) -// Attachment -> -// IO (Promise Attachment) -export const loadData = () => { - return async (attachment: any) => { - if (!isValid(attachment)) { - throw new TypeError("'attachment' is not valid"); - } +export const loadData = async (attachment: any) => { + if (!isValid(attachment)) { + throw new TypeError("'attachment' is not valid"); + } - const isAlreadyLoaded = hasData(attachment); + const isAlreadyLoaded = hasData(attachment); - if (isAlreadyLoaded) { - return attachment; - } + if (isAlreadyLoaded) { + return attachment; + } - if (!isString(attachment.path)) { - throw new TypeError("'attachment.path' is required"); - } + if (!isString(attachment.path)) { + throw new TypeError("'attachment.path' is required"); + } - const data = await readAttachmentData(attachment.path); - return { ...attachment, data }; - }; + const data = await readAttachmentData(attachment.path); + return { ...attachment, data }; }; // deleteData :: (RelativePath -> IO Unit) diff --git a/ts/util/attachmentsUtil.ts b/ts/util/attachmentsUtil.ts index 3c8ebfea1..40e700e94 100644 --- a/ts/util/attachmentsUtil.ts +++ b/ts/util/attachmentsUtil.ts @@ -157,10 +157,6 @@ export async function autoScale( blob.size <= maxSize && !makeSquare ) { - readAndResizedBlob = dataURLToBlob( - (canvas.image as HTMLCanvasElement).toDataURL('image/jpeg', 1) - ); - // the canvas has a size of whatever was given by the caller of autoscale(). // so we have to return those measures as the loaded file has now those measures. return { @@ -223,6 +219,7 @@ export async function getFileAndStoreLocally( maxMeasurements ); + // this operation might change the file size, so be sure to rely on it on return here. const attachmentSavedLocally = await processNewAttachment({ data: await scaled.blob.arrayBuffer(), contentType: attachment.contentType, @@ -242,7 +239,7 @@ export async function getFileAndStoreLocally( height: scaled.height, screenshot: null, thumbnail: null, - size: scaled.blob.size, + size: attachmentSavedLocally.size, // url: undefined, flags: attachmentFlags || undefined,