From cdc8956fa2aa1d3454322c6294f0816a55f3642e Mon Sep 17 00:00:00 2001 From: Sorunome <mail@sorunome.de> Date: Mon, 5 Mar 2018 14:16:35 +0100 Subject: [PATCH] better postmark parsing and added tests --- package.json | 1 + src/messageprocessor.ts | 121 ++++++++++++++++++++++++---------- test/test_messageprocessor.ts | 73 +++++++++++++++++++- 3 files changed, 158 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 061f123..b1c3fa5 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "command-line-args": "^4.0.1", "command-line-usage": "^4.1.0", "discord.js": "^11.3.0", + "escape-html": "^1.0.3", "escape-string-regexp": "^1.0.5", "js-yaml": "^3.10.0", "marked": "^0.3.15", diff --git a/src/messageprocessor.ts b/src/messageprocessor.ts index cb55148..5ba895a 100644 --- a/src/messageprocessor.ts +++ b/src/messageprocessor.ts @@ -3,16 +3,20 @@ import * as marked from "marked"; import * as log from "npmlog"; import { DiscordBot } from "./bot"; import * as escapeStringRegexp from "escape-string-regexp"; +import * as escapeHtml from "escape-html"; const USER_REGEX = /<@!?([0-9]*)>/g; const USER_REGEX_POSTMARK = /<@!?([0-9]*)>/g; const CHANNEL_REGEX = /<#?([0-9]*)>/g; const CHANNEL_REGEX_POSTMARK = /<#?([0-9]*)>/g; const EMOJI_SIZE = "1em"; -const EMOJI_REGEX = /<:\w+:?([0-9]*)>/g; -const EMOJI_REGEX_POSTMARK = /<:\w+:?([0-9]*)>/g; +const EMOJI_REGEX = /<:(\w+):([0-9]*)>/g; +const EMOJI_REGEX_POSTMARK = /<:(\w+):([0-9]*)>/g; const MATRIX_TO_LINK = "https://matrix.to/#/"; +const NAME_EMOJI_REGEX_GROUP = 1; +const ID_EMOJI_REGEX_GROUP = 2; + marked.setOptions({ sanitize: true, }); @@ -40,29 +44,31 @@ export class MessageProcessor { public async FormatDiscordMessage(msg: Discord.Message): Promise<MessageProcessorMatrixResult> { const result = new MessageProcessorMatrixResult(); - // first do the plain-text body - result.body = await this.InsertDiscordSyntax(msg.content, msg, false); + let content = msg.content; + // embeds are markdown formatted, thus inserted before + // for both plaintext and markdown + content = this.InsertEmbeds(content, msg); + // for the formatted body we need to parse markdown first // as else it'll HTML escape the result of the discord syntax - let content = msg.content; - content = marked(content); - content = await this.InsertDiscordSyntax(content, msg, true); - result.formattedBody = content; + let contentPostmark = marked(content); + + // parse the plain text stuff + content = this.ReplaceMembers(content, msg); + content = this.ReplaceChannels(content, msg); + content = await this.ReplaceEmoji(content, msg); + + // parse postmark stuff + contentPostmark = this.ReplaceMembersPostmark(contentPostmark, msg); + contentPostmark = this.ReplaceChannelsPostmark(contentPostmark, msg); + contentPostmark = await this.ReplaceEmojiPostmark(contentPostmark, msg); + + result.body = content; + result.formattedBody = contentPostmark; return result; } - public async InsertDiscordSyntax(content: string, msg: Discord.Message, postmark: boolean): Promise<string> { - // Replace embeds. - content = this.InsertEmbeds(content, msg); - - // Replace Users - content = this.ReplaceMembers(content, msg, postmark); - content = this.ReplaceChannels(content, msg, postmark); - content = await this.ReplaceEmoji(content, msg, postmark); - return content; - } - public InsertEmbeds(content: string, msg: Discord.Message): string { for (const embed of msg.embeds) { let embedContent = "\n\n----"; // Horizontal rule. Two to make sure the content doesn't become a title. @@ -78,49 +84,96 @@ export class MessageProcessor { return content; } - public ReplaceMembers(content: string, msg: Discord.Message, postmark: boolean = false): string { - const reg = postmark ? USER_REGEX_POSTMARK : USER_REGEX; - let results = reg.exec(content); + public ReplaceMembers(content: string, msg: Discord.Message): string { + let results = USER_REGEX.exec(content); while (results !== null) { const id = results[1]; const member = msg.guild.members.get(id); const memberId = `@_discord_${id}:${this.opts.domain}`; const memberStr = member ? member.user.username : memberId; content = content.replace(results[0], memberStr); - results = reg.exec(content); + results = USER_REGEX.exec(content); + } + return content; + } + + public ReplaceMembersPostmark(content: string, msg: Discord.Message): string { + let results = USER_REGEX_POSTMARK.exec(content); + while (results !== null) { + const id = results[1]; + const member = msg.guild.members.get(id); + const memberId = escapeHtml(`@_discord_${id}:${this.opts.domain}`); + let memberName = memberId; + if (member) { + memberName = escapeHtml(member.user.username); + } + const memberStr = `<a href="${MATRIX_TO_LINK}${memberId}">${memberName}</a>`; + content = content.replace(results[0], memberStr); + results = USER_REGEX_POSTMARK.exec(content); } return content; } - public ReplaceChannels(content: string, msg: Discord.Message, postmark: boolean = false): string { - const reg = postmark ? CHANNEL_REGEX_POSTMARK : CHANNEL_REGEX; - let results = reg.exec(content); + public ReplaceChannels(content: string, msg: Discord.Message): string { + let results = CHANNEL_REGEX.exec(content); while (results !== null) { const id = results[1]; const channel = msg.guild.channels.get(id); - const roomId = `#_discord_${msg.guild.id}_${id}:${this.opts.domain}`; const channelStr = channel ? "#" + channel.name : "#" + id; - content = content.replace(results[0], `[${channelStr}](${MATRIX_TO_LINK}${roomId})`); - results = reg.exec(content); + content = content.replace(results[0], channelStr); + results = CHANNEL_REGEX.exec(content); } return content; } - public async ReplaceEmoji(content: string, msg: Discord.Message, postmark: boolean = false): Promise<string> { - const reg = postmark ? EMOJI_REGEX_POSTMARK : EMOJI_REGEX; - let results = reg.exec(content); + public ReplaceChannelsPostmark(content: string, msg: Discord.Message): string { + let results = CHANNEL_REGEX_POSTMARK.exec(content); while (results !== null) { const id = results[1]; + const channel = msg.guild.channels.get(id); + const roomId = escapeHtml(`#_discord_${msg.guild.id}_${id}:${this.opts.domain}`); + const channelStr = escapeHtml(channel ? "#" + channel.name : "#" + id); + const replaceStr = `<a href="${MATRIX_TO_LINK}${roomId}">${channelStr}</a>`; + content = content.replace(results[0], replaceStr); + results = CHANNEL_REGEX_POSTMARK.exec(content); + } + return content; + } + + public async ReplaceEmoji(content: string, msg: Discord.Message): Promise<string> { + let results = EMOJI_REGEX.exec(content); + while (results !== null) { + const name = results[NAME_EMOJI_REGEX_GROUP]; + const id = results[ID_EMOJI_REGEX_GROUP]; + try { + // we still fetch the mxcUrl to check if the emoji is valid + const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id); + content = content.replace(results[0], `:${name}:`); + } catch (ex) { + log.warn("MessageProcessor", + `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`, + ); + } + results = EMOJI_REGEX.exec(content); + } + return content; + } + + public async ReplaceEmojiPostmark(content: string, msg: Discord.Message): Promise<string> { + let results = EMOJI_REGEX_POSTMARK.exec(content); + while (results !== null) { + const name = escapeHtml(results[NAME_EMOJI_REGEX_GROUP]); + const id = results[ID_EMOJI_REGEX_GROUP]; try { const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id); content = content.replace(results[0], - `<img alt="${id}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`); + `<img alt="${name}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`); } catch (ex) { log.warn("MessageProcessor", `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`, ); } - results = reg.exec(content); + results = EMOJI_REGEX_POSTMARK.exec(content); } return content; } diff --git a/test/test_messageprocessor.ts b/test/test_messageprocessor.ts index 9a678ad..2dc07b8 100644 --- a/test/test_messageprocessor.ts +++ b/test/test_messageprocessor.ts @@ -72,6 +72,29 @@ describe("MessageProcessor", () => { Chai.assert.equal(content, "Hello TestUsername"); }); }); + describe("ReplaceMembersPostmark", () => { + it("processes members missing from the guild correctly", () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + const channel = new Discord.TextChannel(guild, null); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <@!12345>"; + content = processor.ReplaceMembersPostmark(content, msg); + Chai.assert.equal(content, + "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">@_discord_12345:localhost</a>"); + }); + it("processes members with usernames correctly", () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + guild._mockAddMember(new MockMember("12345", "TestUsername")); + const channel = new Discord.TextChannel(guild, null); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <@!12345>"; + content = processor.ReplaceMembersPostmark(content, msg); + Chai.assert.equal(content, + "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">TestUsername</a>"); + }); + }); describe("ReplaceChannels", () => { it("processes unknown channel correctly", () => { const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); @@ -80,7 +103,7 @@ describe("MessageProcessor", () => { const msg = new Discord.Message(channel, null, null); let content = "Hello <#123456789>"; content = processor.ReplaceChannels(content, msg); - Chai.assert.equal(content, "Hello [#123456789](https://matrix.to/#/#_discord_123_123456789:localhost)"); + Chai.assert.equal(content, "Hello #123456789"); }); it("processes channels correctly", () => { const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); @@ -90,7 +113,30 @@ describe("MessageProcessor", () => { const msg = new Discord.Message(channel, null, null); let content = "Hello <#456>"; content = processor.ReplaceChannels(content, msg); - Chai.assert.equal(content, "Hello [#TestChannel](https://matrix.to/#/#_discord_123_456:localhost)"); + Chai.assert.equal(content, "Hello #TestChannel"); + }); + }); + describe("ReplaceChannelsPostmark", () => { + it("processes unknown channel correctly", () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"}); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <#123456789>"; + content = processor.ReplaceChannelsPostmark(content, msg); + Chai.assert.equal(content, + "Hello <a href=\"https://matrix.to/#/#_discord_123_123456789:localhost\">#123456789</a>"); + }); + it("processes channels correctly", () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"}); + guild.channels.set("456", channel); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <#456>"; + content = processor.ReplaceChannelsPostmark(content, msg); + Chai.assert.equal(content, + "Hello <a href=\"https://matrix.to/#/#_discord_123_456:localhost\">#TestChannel</a>"); }); }); describe("ReplaceEmoji", () => { @@ -111,7 +157,28 @@ describe("MessageProcessor", () => { const msg = new Discord.Message(channel, null, null); let content = "Hello <:hello:3333333>"; content = await processor.ReplaceEmoji(content, msg); - Chai.assert.equal(content, "Hello <img alt=\"3333333\" src=\"mxc://image\" style=\"height: 1em;\"/>"); + Chai.assert.equal(content, "Hello :hello:"); + }); + }); + describe("ReplaceEmojiPostmark", () => { + it("processes unknown emoji correctly", async () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"}); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <:hello:123456789>"; + content = await processor.ReplaceEmojiPostmark(content, msg); + Chai.assert.equal(content, "Hello <:hello:123456789>"); + }); + it("processes emoji correctly", async () => { + const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot); + const guild: any = new MockGuild("123", []); + const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"}); + guild.channels.set("456", channel); + const msg = new Discord.Message(channel, null, null); + let content = "Hello <:hello:3333333>"; + content = await processor.ReplaceEmojiPostmark(content, msg); + Chai.assert.equal(content, "Hello <img alt=\"hello\" src=\"mxc://image\" style=\"height: 1em;\"/>"); }); }); describe("FindMentionsInPlainBody", () => { -- GitLab