From cdc8956fa2aa1d3454322c6294f0816a55f3642e Mon Sep 17 00:00:00 2001
From: Sorunome <mail@sorunome.de>
Date: Mon, 5 Mar 2018 14:16:35 +0100
Subject: [PATCH] better postmark parsing and added tests

---
 package.json                  |   1 +
 src/messageprocessor.ts       | 121 ++++++++++++++++++++++++----------
 test/test_messageprocessor.ts |  73 +++++++++++++++++++-
 3 files changed, 158 insertions(+), 37 deletions(-)

diff --git a/package.json b/package.json
index 061f123..b1c3fa5 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
     "command-line-args": "^4.0.1",
     "command-line-usage": "^4.1.0",
     "discord.js": "^11.3.0",
+    "escape-html": "^1.0.3",
     "escape-string-regexp": "^1.0.5",
     "js-yaml": "^3.10.0",
     "marked": "^0.3.15",
diff --git a/src/messageprocessor.ts b/src/messageprocessor.ts
index cb55148..5ba895a 100644
--- a/src/messageprocessor.ts
+++ b/src/messageprocessor.ts
@@ -3,16 +3,20 @@ import * as marked from "marked";
 import * as log from "npmlog";
 import { DiscordBot } from "./bot";
 import * as escapeStringRegexp from "escape-string-regexp";
+import * as escapeHtml from "escape-html";
 
 const USER_REGEX = /<@!?([0-9]*)>/g;
 const USER_REGEX_POSTMARK = /&lt;@!?([0-9]*)&gt;/g;
 const CHANNEL_REGEX = /<#?([0-9]*)>/g;
 const CHANNEL_REGEX_POSTMARK = /&lt;#?([0-9]*)&gt;/g;
 const EMOJI_SIZE = "1em";
-const EMOJI_REGEX = /<:\w+:?([0-9]*)>/g;
-const EMOJI_REGEX_POSTMARK = /&lt;:\w+:?([0-9]*)&gt;/g;
+const EMOJI_REGEX = /<:(\w+):([0-9]*)>/g;
+const EMOJI_REGEX_POSTMARK = /&lt;:(\w+):([0-9]*)&gt;/g;
 const MATRIX_TO_LINK = "https://matrix.to/#/";
 
+const NAME_EMOJI_REGEX_GROUP = 1;
+const ID_EMOJI_REGEX_GROUP = 2;
+
 marked.setOptions({
     sanitize: true,
 });
@@ -40,29 +44,31 @@ export class MessageProcessor {
 
     public async FormatDiscordMessage(msg: Discord.Message): Promise<MessageProcessorMatrixResult> {
         const result = new MessageProcessorMatrixResult();
-        // first do the plain-text body
-        result.body = await this.InsertDiscordSyntax(msg.content, msg, false);
 
+        let content = msg.content;
+        // embeds are markdown formatted, thus inserted before
+        // for both plaintext and markdown
+        content = this.InsertEmbeds(content, msg);
+        
         // for the formatted body we need to parse markdown first
         // as else it'll HTML escape the result of the discord syntax
-        let content = msg.content;
-        content = marked(content);
-        content = await this.InsertDiscordSyntax(content, msg, true);
-        result.formattedBody = content;
+        let contentPostmark = marked(content);
+        
+        // parse the plain text stuff
+        content = this.ReplaceMembers(content, msg);
+        content = this.ReplaceChannels(content, msg);
+        content = await this.ReplaceEmoji(content, msg);
+        
+        // parse postmark stuff
+        contentPostmark = this.ReplaceMembersPostmark(contentPostmark, msg);
+        contentPostmark = this.ReplaceChannelsPostmark(contentPostmark, msg);
+        contentPostmark = await this.ReplaceEmojiPostmark(contentPostmark, msg);
+        
+        result.body = content;
+        result.formattedBody = contentPostmark;
         return result;
     }
 
-    public async InsertDiscordSyntax(content: string, msg: Discord.Message, postmark: boolean): Promise<string> {
-        // Replace embeds.
-        content = this.InsertEmbeds(content, msg);
-
-        // Replace Users
-        content = this.ReplaceMembers(content, msg, postmark);
-        content = this.ReplaceChannels(content, msg, postmark);
-        content = await this.ReplaceEmoji(content, msg, postmark);
-        return content;
-    }
-
     public InsertEmbeds(content: string, msg: Discord.Message): string {
         for (const embed of msg.embeds) {
             let embedContent = "\n\n----"; // Horizontal rule. Two to make sure the content doesn't become a title.
@@ -78,49 +84,96 @@ export class MessageProcessor {
         return content;
     }
 
-    public ReplaceMembers(content: string, msg: Discord.Message, postmark: boolean = false): string {
-        const reg = postmark ? USER_REGEX_POSTMARK : USER_REGEX;
-        let results = reg.exec(content);
+    public ReplaceMembers(content: string, msg: Discord.Message): string {
+        let results = USER_REGEX.exec(content);
         while (results !== null) {
             const id = results[1];
             const member = msg.guild.members.get(id);
             const memberId = `@_discord_${id}:${this.opts.domain}`;
             const memberStr = member ? member.user.username : memberId;
             content = content.replace(results[0], memberStr);
-            results = reg.exec(content);
+            results = USER_REGEX.exec(content);
+        }
+        return content;
+    }
+
+    public ReplaceMembersPostmark(content: string, msg: Discord.Message): string {
+        let results = USER_REGEX_POSTMARK.exec(content);
+        while (results !== null) {
+            const id = results[1];
+            const member = msg.guild.members.get(id);
+            const memberId = escapeHtml(`@_discord_${id}:${this.opts.domain}`);
+            let memberName = memberId;
+            if (member) {
+                memberName = escapeHtml(member.user.username);
+            }
+            const memberStr = `<a href="${MATRIX_TO_LINK}${memberId}">${memberName}</a>`;
+            content = content.replace(results[0], memberStr);
+            results = USER_REGEX_POSTMARK.exec(content);
         }
         return content;
     }
 
-    public ReplaceChannels(content: string, msg: Discord.Message, postmark: boolean = false): string {
-        const reg = postmark ? CHANNEL_REGEX_POSTMARK : CHANNEL_REGEX;
-        let results = reg.exec(content);
+    public ReplaceChannels(content: string, msg: Discord.Message): string {
+        let results = CHANNEL_REGEX.exec(content);
         while (results !== null) {
             const id = results[1];
             const channel = msg.guild.channels.get(id);
-            const roomId = `#_discord_${msg.guild.id}_${id}:${this.opts.domain}`;
             const channelStr = channel ? "#" + channel.name : "#" + id;
-            content = content.replace(results[0], `[${channelStr}](${MATRIX_TO_LINK}${roomId})`);
-            results = reg.exec(content);
+            content = content.replace(results[0], channelStr);
+            results = CHANNEL_REGEX.exec(content);
         }
         return content;
     }
 
-    public async ReplaceEmoji(content: string, msg: Discord.Message, postmark: boolean = false): Promise<string> {
-        const reg = postmark ? EMOJI_REGEX_POSTMARK : EMOJI_REGEX;
-        let results = reg.exec(content);
+    public ReplaceChannelsPostmark(content: string, msg: Discord.Message): string {
+        let results = CHANNEL_REGEX_POSTMARK.exec(content);
         while (results !== null) {
             const id = results[1];
+            const channel = msg.guild.channels.get(id);
+            const roomId = escapeHtml(`#_discord_${msg.guild.id}_${id}:${this.opts.domain}`);
+            const channelStr = escapeHtml(channel ? "#" + channel.name : "#" + id);
+            const replaceStr = `<a href="${MATRIX_TO_LINK}${roomId}">${channelStr}</a>`;
+            content = content.replace(results[0], replaceStr);
+            results = CHANNEL_REGEX_POSTMARK.exec(content);
+        }
+        return content;
+    }
+
+    public async ReplaceEmoji(content: string, msg: Discord.Message): Promise<string> {
+        let results = EMOJI_REGEX.exec(content);
+        while (results !== null) {
+            const name = results[NAME_EMOJI_REGEX_GROUP];
+            const id = results[ID_EMOJI_REGEX_GROUP];
+            try {
+                // we still fetch the mxcUrl to check if the emoji is valid
+                const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id);
+                content = content.replace(results[0], `:${name}:`);
+            } catch (ex) {
+                log.warn("MessageProcessor",
+                    `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`,
+                );
+            }
+            results = EMOJI_REGEX.exec(content);
+        }
+        return content;
+    }
+
+    public async ReplaceEmojiPostmark(content: string, msg: Discord.Message): Promise<string> {
+        let results = EMOJI_REGEX_POSTMARK.exec(content);
+        while (results !== null) {
+            const name = escapeHtml(results[NAME_EMOJI_REGEX_GROUP]);
+            const id = results[ID_EMOJI_REGEX_GROUP];
             try {
                 const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id);
                 content = content.replace(results[0],
-                    `<img alt="${id}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`);
+                    `<img alt="${name}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`);
             } catch (ex) {
                 log.warn("MessageProcessor",
                     `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`,
                 );
             }
-            results = reg.exec(content);
+            results = EMOJI_REGEX_POSTMARK.exec(content);
         }
         return content;
     }
diff --git a/test/test_messageprocessor.ts b/test/test_messageprocessor.ts
index 9a678ad..2dc07b8 100644
--- a/test/test_messageprocessor.ts
+++ b/test/test_messageprocessor.ts
@@ -72,6 +72,29 @@ describe("MessageProcessor", () => {
             Chai.assert.equal(content, "Hello TestUsername");
         });
     });
+    describe("ReplaceMembersPostmark", () => {
+        it("processes members missing from the guild correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, null);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;@!12345&gt;";
+            content = processor.ReplaceMembersPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">@_discord_12345:localhost</a>");
+        });
+        it("processes members with usernames correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            guild._mockAddMember(new MockMember("12345", "TestUsername"));
+            const channel = new Discord.TextChannel(guild, null);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;@!12345&gt;";
+            content = processor.ReplaceMembersPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">TestUsername</a>");
+        });
+    });
     describe("ReplaceChannels", () => {
         it("processes unknown channel correctly", () => {
             const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
@@ -80,7 +103,7 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <#123456789>";
             content = processor.ReplaceChannels(content, msg);
-            Chai.assert.equal(content, "Hello [#123456789](https://matrix.to/#/#_discord_123_123456789:localhost)");
+            Chai.assert.equal(content, "Hello #123456789");
         });
         it("processes channels correctly", () => {
             const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
@@ -90,7 +113,30 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <#456>";
             content = processor.ReplaceChannels(content, msg);
-            Chai.assert.equal(content, "Hello [#TestChannel](https://matrix.to/#/#_discord_123_456:localhost)");
+            Chai.assert.equal(content, "Hello #TestChannel");
+        });
+    });
+    describe("ReplaceChannelsPostmark", () => {
+        it("processes unknown channel correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;#123456789&gt;";
+            content = processor.ReplaceChannelsPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/#_discord_123_123456789:localhost\">#123456789</a>");
+        });
+        it("processes channels correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            guild.channels.set("456", channel);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;#456&gt;";
+            content = processor.ReplaceChannelsPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/#_discord_123_456:localhost\">#TestChannel</a>");
         });
     });
     describe("ReplaceEmoji", () => {
@@ -111,7 +157,28 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <:hello:3333333>";
             content = await processor.ReplaceEmoji(content, msg);
-            Chai.assert.equal(content, "Hello <img alt=\"3333333\" src=\"mxc://image\" style=\"height: 1em;\"/>");
+            Chai.assert.equal(content, "Hello :hello:");
+        });
+    });
+    describe("ReplaceEmojiPostmark", () => {
+        it("processes unknown emoji correctly", async () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;:hello:123456789&gt;";
+            content = await processor.ReplaceEmojiPostmark(content, msg);
+            Chai.assert.equal(content, "Hello &lt;:hello:123456789&gt;");
+        });
+        it("processes emoji correctly", async () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            guild.channels.set("456", channel);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;:hello:3333333&gt;";
+            content = await processor.ReplaceEmojiPostmark(content, msg);
+            Chai.assert.equal(content, "Hello <img alt=\"hello\" src=\"mxc://image\" style=\"height: 1em;\"/>");
         });
     });
     describe("FindMentionsInPlainBody", () => {
-- 
GitLab