Merge pull request #105 from Sorunome/develop

fix discord message content parsing

package.json

@@ -38,6 +38,7 @@
     "command-line-args": "^4.0.1",
     "command-line-usage": "^4.1.0",
     "discord.js": "^11.3.0",
+    "escape-html": "^1.0.3",
     "escape-string-regexp": "^1.0.5",
     "js-yaml": "^3.10.0",
     "marked": "^0.3.15",

src/messageprocessor.ts

@@ -3,13 +3,20 @@ import * as marked from "marked";
 import * as log from "npmlog";
 import { DiscordBot } from "./bot";
 import * as escapeStringRegexp from "escape-string-regexp";
+import * as escapeHtml from "escape-html";
 
 const USER_REGEX = /<@!?([0-9]*)>/g;
+const USER_REGEX_POSTMARK = /&lt;@!?([0-9]*)&gt;/g;
 const CHANNEL_REGEX = /<#?([0-9]*)>/g;
+const CHANNEL_REGEX_POSTMARK = /&lt;#?([0-9]*)&gt;/g;
 const EMOJI_SIZE = "1em";
-const EMOJI_REGEX = /<:\w+:?([0-9]*)>/g;
+const EMOJI_REGEX = /<:(\w+):([0-9]*)>/g;
+const EMOJI_REGEX_POSTMARK = /&lt;:(\w+):([0-9]*)&gt;/g;
 const MATRIX_TO_LINK = "https://matrix.to/#/";
 
+const NAME_EMOJI_REGEX_GROUP = 1;
+const ID_EMOJI_REGEX_GROUP = 2;
+
 marked.setOptions({
     sanitize: true,
 });
@@ -37,17 +44,28 @@ export class MessageProcessor {
 
     public async FormatDiscordMessage(msg: Discord.Message): Promise<MessageProcessorMatrixResult> {
         const result = new MessageProcessorMatrixResult();
-        // Replace embeds.
+
         let content = msg.content;
+        // embeds are markdown formatted, thus inserted before
+        // for both plaintext and markdown
         content = this.InsertEmbeds(content, msg);
         
-        // Replace Users
+        // for the formatted body we need to parse markdown first
+        // as else it'll HTML escape the result of the discord syntax
+        let contentPostmark = marked(content);
+        
+        // parse the plain text stuff
         content = this.ReplaceMembers(content, msg);
         content = this.ReplaceChannels(content, msg);
         content = await this.ReplaceEmoji(content, msg);
-        // Replace channels
+        
+        // parse postmark stuff
+        contentPostmark = this.ReplaceMembersPostmark(contentPostmark, msg);
+        contentPostmark = this.ReplaceChannelsPostmark(contentPostmark, msg);
+        contentPostmark = await this.ReplaceEmojiPostmark(contentPostmark, msg);
+        
         result.body = content;
-        result.formattedBody = marked(content);
+        result.formattedBody = contentPostmark;
         return result;
     }
 
@@ -79,27 +97,58 @@ export class MessageProcessor {
         return content;
     }
 
+    public ReplaceMembersPostmark(content: string, msg: Discord.Message): string {
+        let results = USER_REGEX_POSTMARK.exec(content);
+        while (results !== null) {
+            const id = results[1];
+            const member = msg.guild.members.get(id);
+            const memberId = escapeHtml(`@_discord_${id}:${this.opts.domain}`);
+            let memberName = memberId;
+            if (member) {
+                memberName = escapeHtml(member.user.username);
+            }
+            const memberStr = `<a href="${MATRIX_TO_LINK}${memberId}">${memberName}</a>`;
+            content = content.replace(results[0], memberStr);
+            results = USER_REGEX_POSTMARK.exec(content);
+        }
+        return content;
+    }
+
     public ReplaceChannels(content: string, msg: Discord.Message): string {
         let results = CHANNEL_REGEX.exec(content);
         while (results !== null) {
             const id = results[1];
             const channel = msg.guild.channels.get(id);
-          const roomId = `#_discord_${msg.guild.id}_${id}:${this.opts.domain}`;
             const channelStr = channel ? "#" + channel.name : "#" + id;
-          content = content.replace(results[0], `[${channelStr}](${MATRIX_TO_LINK}${roomId})`);
+            content = content.replace(results[0], channelStr);
             results = CHANNEL_REGEX.exec(content);
         }
         return content;
     }
 
+    public ReplaceChannelsPostmark(content: string, msg: Discord.Message): string {
+        let results = CHANNEL_REGEX_POSTMARK.exec(content);
+        while (results !== null) {
+            const id = results[1];
+            const channel = msg.guild.channels.get(id);
+            const roomId = escapeHtml(`#_discord_${msg.guild.id}_${id}:${this.opts.domain}`);
+            const channelStr = escapeHtml(channel ? "#" + channel.name : "#" + id);
+            const replaceStr = `<a href="${MATRIX_TO_LINK}${roomId}">${channelStr}</a>`;
+            content = content.replace(results[0], replaceStr);
+            results = CHANNEL_REGEX_POSTMARK.exec(content);
+        }
+        return content;
+    }
+
     public async ReplaceEmoji(content: string, msg: Discord.Message): Promise<string> {
         let results = EMOJI_REGEX.exec(content);
         while (results !== null) {
-          const id = results[1];
+            const name = results[NAME_EMOJI_REGEX_GROUP];
+            const id = results[ID_EMOJI_REGEX_GROUP];
             try {
+                // we still fetch the mxcUrl to check if the emoji is valid
                 const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id);
-              content = content.replace(results[0],
-                  `<img alt="${id}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`);
+                content = content.replace(results[0], `:${name}:`);
             } catch (ex) {
                 log.warn("MessageProcessor",
                     `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`,
@@ -110,6 +159,25 @@ export class MessageProcessor {
         return content;
     }
 
+    public async ReplaceEmojiPostmark(content: string, msg: Discord.Message): Promise<string> {
+        let results = EMOJI_REGEX_POSTMARK.exec(content);
+        while (results !== null) {
+            const name = escapeHtml(results[NAME_EMOJI_REGEX_GROUP]);
+            const id = results[ID_EMOJI_REGEX_GROUP];
+            try {
+                const mxcUrl = await this.bot.GetGuildEmoji(msg.guild, id);
+                content = content.replace(results[0],
+                    `<img alt="${name}" src="${mxcUrl}" style="height: ${EMOJI_SIZE};"/>`);
+            } catch (ex) {
+                log.warn("MessageProcessor",
+                    `Could not insert emoji ${id} for msg ${msg.id} in guild ${msg.guild.id}: ${ex}`,
+                );
+            }
+            results = EMOJI_REGEX_POSTMARK.exec(content);
+        }
+        return content;
+    }
+
     public FindMentionsInPlainBody(body: string, members: Discord.GuildMember[]): string {
       for (const member of members) {
         const matcher = escapeStringRegexp(member.user.username + "#" + member.user.discriminator) + "|" +

test/test_messageprocessor.ts

@@ -72,6 +72,29 @@ describe("MessageProcessor", () => {
             Chai.assert.equal(content, "Hello TestUsername");
         });
     });
+    describe("ReplaceMembersPostmark", () => {
+        it("processes members missing from the guild correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, null);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;@!12345&gt;";
+            content = processor.ReplaceMembersPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">@_discord_12345:localhost</a>");
+        });
+        it("processes members with usernames correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            guild._mockAddMember(new MockMember("12345", "TestUsername"));
+            const channel = new Discord.TextChannel(guild, null);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;@!12345&gt;";
+            content = processor.ReplaceMembersPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/@_discord_12345:localhost\">TestUsername</a>");
+        });
+    });
     describe("ReplaceChannels", () => {
         it("processes unknown channel correctly", () => {
             const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
@@ -80,7 +103,7 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <#123456789>";
             content = processor.ReplaceChannels(content, msg);
-            Chai.assert.equal(content, "Hello [#123456789](https://matrix.to/#/#_discord_123_123456789:localhost)");
+            Chai.assert.equal(content, "Hello #123456789");
         });
         it("processes channels correctly", () => {
             const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
@@ -90,7 +113,30 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <#456>";
             content = processor.ReplaceChannels(content, msg);
-            Chai.assert.equal(content, "Hello [#TestChannel](https://matrix.to/#/#_discord_123_456:localhost)");
+            Chai.assert.equal(content, "Hello #TestChannel");
+        });
+    });
+    describe("ReplaceChannelsPostmark", () => {
+        it("processes unknown channel correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;#123456789&gt;";
+            content = processor.ReplaceChannelsPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/#_discord_123_123456789:localhost\">#123456789</a>");
+        });
+        it("processes channels correctly", () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            guild.channels.set("456", channel);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;#456&gt;";
+            content = processor.ReplaceChannelsPostmark(content, msg);
+            Chai.assert.equal(content,
+                "Hello <a href=\"https://matrix.to/#/#_discord_123_456:localhost\">#TestChannel</a>");
         });
     });
     describe("ReplaceEmoji", () => {
@@ -111,7 +157,28 @@ describe("MessageProcessor", () => {
             const msg = new Discord.Message(channel, null, null);
             let content = "Hello <:hello:3333333>";
             content = await processor.ReplaceEmoji(content, msg);
-            Chai.assert.equal(content, "Hello <img alt=\"3333333\" src=\"mxc://image\" style=\"height: 1em;\"/>");
+            Chai.assert.equal(content, "Hello :hello:");
+        });
+    });
+    describe("ReplaceEmojiPostmark", () => {
+        it("processes unknown emoji correctly", async () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;:hello:123456789&gt;";
+            content = await processor.ReplaceEmojiPostmark(content, msg);
+            Chai.assert.equal(content, "Hello &lt;:hello:123456789&gt;");
+        });
+        it("processes emoji correctly", async () => {
+            const processor = new MessageProcessor(new MessageProcessorOpts("localhost"), <DiscordBot> bot);
+            const guild: any = new MockGuild("123", []);
+            const channel = new Discord.TextChannel(guild, {id: "456", name: "TestChannel"});
+            guild.channels.set("456", channel);
+            const msg = new Discord.Message(channel, null, null);
+            let content = "Hello &lt;:hello:3333333&gt;";
+            content = await processor.ReplaceEmojiPostmark(content, msg);
+            Chai.assert.equal(content, "Hello <img alt=\"hello\" src=\"mxc://image\" style=\"height: 1em;\"/>");
         });
     });
     describe("FindMentionsInPlainBody", () => {