web/composer: send inline URL previews

Signed-off-by: Sumner Evans <[email protected]>

pkg/gomuks/media.go

@@ -348,22 +348,71 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) {
 
 func (gmx *Gomuks) UploadMedia(w http.ResponseWriter, r *http.Request) {
 	log := hlog.FromRequest(r)
-	tempFile, err := os.CreateTemp(gmx.TempDir, "upload-*")
+	encrypt, _ := strconv.ParseBool(r.URL.Query().Get("encrypt"))
+	content, err := gmx.cacheAndUploadMedia(r.Context(), r.Body, encrypt, r.URL.Query().Get("filename"))
 	if err != nil {
-		log.Err(err).Msg("Failed to create temporary file")
-		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to create temp file: %v", err)).Write(w)
+		log.Err(err).Msg("Failed to upload media")
+		writeMaybeRespError(err, w)
+		return
+	}
+	exhttp.WriteJSONResponse(w, http.StatusOK, content)
+}
+
+func (gmx *Gomuks) GetURLPreview(w http.ResponseWriter, r *http.Request) {
+	log := hlog.FromRequest(r)
+	url := r.URL.Query().Get("url")
+	if url == "" {
+		mautrix.MInvalidParam.WithMessage("URL must be provided to preview").Write(w)
 		return
 	}
+	linkPreview, err := gmx.Client.Client.GetURLPreview(r.Context(), url)
+	if err != nil {
+		log.Err(err).Msg("Failed to get URL preview")
+		writeMaybeRespError(err, w)
+		return
+	}
+
+	preview := event.BeeperLinkPreview{
+		LinkPreview: *linkPreview,
+		MatchedURL:  url,
+	}
+
+	if preview.ImageURL != "" {
+		resp, err := gmx.Client.Client.Download(r.Context(), preview.ImageURL.ParseOrIgnore())
+		if err != nil {
+			log.Err(err).Msg("Failed to download URL preview image")
+			writeMaybeRespError(err, w)
+			return
+		}
+		encrypt, _ := strconv.ParseBool(r.URL.Query().Get("encrypt"))
+		defer resp.Body.Close()
+		content, err := gmx.cacheAndUploadMedia(r.Context(), resp.Body, encrypt, "")
+		if err != nil {
+			log.Err(err).Msg("Failed to upload URL preview image")
+			writeMaybeRespError(err, w)
+			return
+		}
+		preview.ImageURL = content.URL
+		preview.ImageEncryption = content.File
+	}
+
+	exhttp.WriteJSONResponse(w, http.StatusOK, preview)
+}
+
+func (gmx *Gomuks) cacheAndUploadMedia(ctx context.Context, reader io.Reader, encrypt bool, fileName string) (*event.MessageEventContent, error) {
+	log := zerolog.Ctx(ctx)
+	tempFile, err := os.CreateTemp(gmx.TempDir, "upload-*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp file %w", err)
+	}
 	defer func() {
 		_ = tempFile.Close()
 		_ = os.Remove(tempFile.Name())
 	}()
 	hasher := sha256.New()
-	_, err = io.Copy(tempFile, io.TeeReader(r.Body, hasher))
+	_, err = io.Copy(tempFile, io.TeeReader(reader, hasher))
 	if err != nil {
-		log.Err(err).Msg("Failed to copy upload media to temporary file")
-		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to copy media to temp file: %v", err)).Write(w)
-		return
+		return nil, fmt.Errorf("failed to copy upload media to temp file: %w", err)
 	}
 	_ = tempFile.Close()
 
@@ -374,39 +423,29 @@ func (gmx *Gomuks) UploadMedia(w http.ResponseWriter, r *http.Request) {
 	} else {
 		err = os.MkdirAll(filepath.Dir(cachePath), 0700)
 		if err != nil {
-			log.Err(err).Msg("Failed to create cache directory")
-			mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to create cache directory: %v", err)).Write(w)
-			return
+			return nil, fmt.Errorf("failed to create cache directory: %w", err)
 		}
 		err = os.Rename(tempFile.Name(), cachePath)
 		if err != nil {
-			log.Err(err).Msg("Failed to rename temporary file")
-			mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to rename temp file: %v", err)).Write(w)
-			return
+			return nil, fmt.Errorf("failed to rename temp file: %w", err)
 		}
 	}
 
 	cacheFile, err := os.Open(cachePath)
 	if err != nil {
-		log.Err(err).Msg("Failed to open cache file")
-		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to open cache file: %v", err)).Write(w)
-		return
+		return nil, fmt.Errorf("failed to open cache file: %w", err)
 	}
 
-	msgType, info, defaultFileName, err := gmx.generateFileInfo(r.Context(), cacheFile)
+	msgType, info, defaultFileName, err := gmx.generateFileInfo(ctx, cacheFile)
 	if err != nil {
-		log.Err(err).Msg("Failed to generate file info")
-		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to generate file info: %v", err)).Write(w)
-		return
+		return nil, fmt.Errorf("failed to generate file info: %w", err)
 	}
-	encrypt, _ := strconv.ParseBool(r.URL.Query().Get("encrypt"))
 	if msgType == event.MsgVideo {
-		err = gmx.generateVideoThumbnail(r.Context(), cacheFile.Name(), encrypt, info)
+		err = gmx.generateVideoThumbnail(ctx, cacheFile.Name(), encrypt, info)
 		if err != nil {
 			log.Warn().Err(err).Msg("Failed to generate video thumbnail")
 		}
 	}
-	fileName := r.URL.Query().Get("filename")
 	if fileName == "" {
 		fileName = defaultFileName
 	}
@@ -416,13 +455,11 @@ func (gmx *Gomuks) UploadMedia(w http.ResponseWriter, r *http.Request) {
 		Info:     info,
 		FileName: fileName,
 	}
-	content.File, content.URL, err = gmx.uploadFile(r.Context(), checksum, cacheFile, encrypt, int64(info.Size), info.MimeType, fileName)
+	content.File, content.URL, err = gmx.uploadFile(ctx, checksum, cacheFile, encrypt, int64(info.Size), info.MimeType, fileName)
 	if err != nil {
-		log.Err(err).Msg("Failed to upload media")
-		writeMaybeRespError(err, w)
-		return
+		return nil, fmt.Errorf("failed to upload media: %w", err)
 	}
-	exhttp.WriteJSONResponse(w, http.StatusOK, content)
+	return content, nil
 }
 
 func (gmx *Gomuks) uploadFile(ctx context.Context, checksum []byte, cacheFile *os.File, encrypt bool, fileSize int64, mimeType, fileName string) (*event.EncryptedFileInfo, id.ContentURIString, error) {

pkg/gomuks/server.go

@@ -54,6 +54,7 @@ func (gmx *Gomuks) CreateAPIRouter() http.Handler {
 	api.HandleFunc("POST /sso", gmx.PrepareSSO)
 	api.HandleFunc("GET /media/{server}/{media_id}", gmx.DownloadMedia)
 	api.HandleFunc("GET /codeblock/{style}", gmx.GetCodeblockCSS)
+	api.HandleFunc("GET /url_preview", gmx.GetURLPreview)
 	return exhttp.ApplyMiddleware(
 		api,
 		hlog.NewHandler(*gmx.Log),

pkg/hicli/json-commands.go

@@ -42,7 +42,7 @@ func (h *HiClient) handleJSONCommand(ctx context.Context, req *JSONCommand) (any
 		})
 	case "send_message":
 		return unmarshalAndCall(req.Data, func(params *sendMessageParams) (*database.Event, error) {
-			return h.SendMessage(ctx, params.RoomID, params.BaseContent, params.Extra, params.Text, params.RelatesTo, params.Mentions)
+			return h.SendMessage(ctx, params.RoomID, params.BaseContent, params.Extra, params.Text, params.RelatesTo, params.Mentions, params.URLPreviews)
 		})
 	case "send_event":
 		return unmarshalAndCall(req.Data, func(params *sendEventParams) (*database.Event, error) {
@@ -215,12 +215,13 @@ type cancelRequestParams struct {
 }
 
 type sendMessageParams struct {
-	RoomID      id.RoomID                  `json:"room_id"`
-	BaseContent *event.MessageEventContent `json:"base_content"`
-	Extra       map[string]any             `json:"extra"`
-	Text        string                     `json:"text"`
-	RelatesTo   *event.RelatesTo           `json:"relates_to"`
-	Mentions    *event.Mentions            `json:"mentions"`
+	RoomID      id.RoomID                   `json:"room_id"`
+	BaseContent *event.MessageEventContent  `json:"base_content"`
+	Extra       map[string]any              `json:"extra"`
+	Text        string                      `json:"text"`
+	RelatesTo   *event.RelatesTo            `json:"relates_to"`
+	Mentions    *event.Mentions             `json:"mentions"`
+	URLPreviews *[]*event.BeeperLinkPreview `json:"url_previews"`
 }
 
 type sendEventParams struct {

pkg/hicli/send.go

@@ -70,6 +70,7 @@ func (h *HiClient) SendMessage(
 	text string,
 	relatesTo *event.RelatesTo,
 	mentions *event.Mentions,
+	urlPreviews *[]*event.BeeperLinkPreview,
 ) (*database.Event, error) {
 	var unencrypted bool
 	if strings.HasPrefix(text, "/unencrypted ") {
@@ -158,6 +159,9 @@ func (h *HiClient) SendMessage(
 		content.MsgType = ""
 		evtType = event.EventSticker
 	}
+	if urlPreviews != nil {
+		content.BeeperLinkPreviews = *urlPreviews
+	}
 	return h.send(ctx, roomID, evtType, &event.Content{Parsed: content, Raw: extra}, origText, unencrypted)
 }
 

web/src/api/rpc.ts

@@ -38,6 +38,7 @@ import type {
 	RoomStateGUID,
 	RoomSummary,
 	TimelineRowID,
+	URLPreview,
 	UserID,
 	UserProfile,
 } from "./types"
@@ -63,6 +64,7 @@ export interface SendMessageParams {
 	media_path?: string
 	relates_to?: RelatesTo
 	mentions?: Mentions
+	url_previews?: URLPreview[]
 }
 
 export default abstract class RPCClient {

web/src/api/types/preferences/preferences.ts

@@ -47,6 +47,12 @@ export const preferences = {
 		allowedContexts: anyContext,
 		defaultValue: true,
 	}),
+	send_bundled_url_previews: new Preference<boolean>({
+		displayName: "Send bundled URL previews",
+		description: "Should bundled URL previews be sent to other users?",
+		allowedContexts: anyContext,
+		defaultValue: true,
+	}),
 	display_read_receipts: new Preference<boolean>({
 		displayName: "Display read receipts",
 		description: "Should read receipts be rendered in the timeline?",

web/src/ui/composer/MessageComposer.css

@@ -80,4 +80,12 @@ div.message-composer {
 			}
 		}
 	}
+
+	> div.url-previews {
+		display: flex;
+		flex-direction: row;
+		gap: 1rem;
+		overflow-x: scroll;
+		margin: 0 0.5rem;
+	}
 }

web/src/ui/composer/MessageComposer.tsx

@@ -24,6 +24,7 @@ import type {
 	MessageEventContent,
 	RelatesTo,
 	RoomID,
+	URLPreview as URLPreviewType,
 } from "@/api/types"
 import { PartialEmoji, emojiToMarkdown } from "@/util/emoji"
 import { isMobileDevice } from "@/util/ismobile.ts"
@@ -36,6 +37,7 @@ import { keyToString } from "../keybindings.ts"
 import { ModalContext } from "../modal"
 import { useRoomContext } from "../roomview/roomcontext.ts"
 import { ReplyBody } from "../timeline/ReplyBody.tsx"
+import URLPreview from "../urlpreview/URLPreview.tsx"
 import type { AutocompleteQuery } from "./Autocompleter.tsx"
 import { ComposerLocation, ComposerLocationValue, ComposerMedia } from "./ComposerMedia.tsx"
 import { charToAutocompleteType, emojiQueryRegex, getAutocompleter } from "./getAutocompleter.ts"
@@ -52,6 +54,7 @@ export interface ComposerState {
 	text: string
 	media: MediaMessageEventContent | null
 	location: ComposerLocationValue | null
+	previews: Record<string, URLPreviewType | "cleared" | null>
 	replyTo: EventID | null
 	silentReply: boolean
 	explicitReplyInThread: boolean
@@ -63,8 +66,9 @@ const MAX_TEXTAREA_ROWS = 10
 const emptyComposer: ComposerState = {
 	text: "",
 	media: null,
-	replyTo: null,
 	location: null,
+	previews: {},
+	replyTo: null,
 	silentReply: false,
 	explicitReplyInThread: false,
 }
@@ -233,6 +237,7 @@ const MessageComposer = () => {
 			text: state.text,
 			relates_to,
 			mentions,
+			url_previews: Object.values(state.previews).filter(p => p !== null && p !== "cleared"),
 		}).catch(err => window.alert("Failed to send message: " + err))
 	}
 	const onComposerCaretChange = (evt: CaretEvent<HTMLTextAreaElement>, newText?: string) => {
@@ -388,6 +393,46 @@ const MessageComposer = () => {
 		}
 		evt.preventDefault()
 	}
+	const resolvePreviews = useCallback((
+		urls: string[],
+		existingPreviews: Record<string, URLPreviewType | "cleared" | null>,
+	) => {
+		const encrypt = !!room.meta.current.encryption_event
+		const previews: Record<string, URLPreviewType | "cleared" | null> = {}
+		let changed = false
+		urls.forEach(url => {
+			if (url.startsWith("https://matrix.to")) return
+
+			if (existingPreviews[url] === undefined) {
+				changed = true
+				previews[url] = null
+				fetch(`_gomuks/url_preview?encrypt=${encrypt}&url=${encodeURIComponent(url)}`, {
+					method: "GET",
+				})
+					.then(async res => {
+						const json = await res.json()
+						if (!res.ok) {
+							throw new Error(json.error)
+						} else {
+							setState(s => ({
+								previews: Object.assign(s.previews, { [url]: json }),
+							}))
+						}
+					})
+					.catch(err => {
+						console.error("Error fetchnig preview for URL", url, err)
+						setState(s => ({
+							previews: Object.assign(s.previews, { [url]: "cleared" }),
+						}))
+					})
+			} else if (existingPreviews[url]) {
+				previews[url] = existingPreviews[url]
+			} else {
+				changed = true
+			}
+		})
+		if (changed) setState({ previews })
+	}, [room.meta])
 	// To ensure the cursor jumps to the end, do this in an effect rather than as the initial value of useState
 	// To try to avoid the input bar flashing, use useLayoutEffect instead of useEffect
 	useLayoutEffect(() => {
@@ -437,6 +482,19 @@ const MessageComposer = () => {
 			draftStore.set(room.roomID, state)
 		}
 	}, [roomCtx, room, state, editing])
+	useEffect(() => {
+		if (!room.preferences.send_bundled_url_previews) {
+			setState({ previews: {}})
+			return
+		}
+		const urls = state.text.matchAll(/\bhttps?:\/\/[^\s/_*]+(?:\/\S*)?\b/gi).map(m => m[0]).toArray()
+		if (!urls.length) {
+			setState({ previews: {}})
+			return
+		}
+		const timeout = setTimeout(() => resolvePreviews(urls, state.previews), 500)
+		return () => clearTimeout(timeout)
+	}, [room.preferences, state.text, state.previews, resolvePreviews])
 	const clearMedia = useCallback(() => setState({ media: null, location: null }), [])
 	const onChangeLocation = useCallback((location: ComposerLocationValue) => setState({ location }), [])
 	const closeReply = useCallback((evt: React.MouseEvent) => {
@@ -594,6 +652,16 @@ const MessageComposer = () => {
 				room={room} client={client}
 				location={state.location} onChange={onChangeLocation} clearLocation={clearMedia}
 			/>}
+			{Object.keys(state.previews).length ? <div className="url-previews">
+				{Object.entries(state.previews).map(([url, preview], i) => {
+					if (preview === "cleared") return null
+					return <URLPreview key={i} url={url} preview={preview}
+						clearPreview={() => setState(s => ({
+							previews: Object.assign(s.previews, { [url]: "cleared" }),
+						}))}
+					/>
+				})}
+			</div> : null}
 			<div className="input-area">
 				{!inlineButtons && <button className="show-more" onClick={openButtonsModal}><MoreIcon/></button>}
 				<textarea

web/src/ui/timeline/TimelineEvent.css

@@ -166,6 +166,13 @@ div.timeline-event {
 			"avatar                       gap                        content  status" auto
 			/ var(--timeline-avatar-size) var(--timeline-avatar-gap) 1fr      var(--timeline-status-size);
 	}
+
+	div.url-previews {
+		display: flex;
+		flex-direction: row;
+		gap: 1rem;
+		overflow-x: scroll;
+	}
 }
 
 div.pinned-event > div.timeline-event {