From 8062ba36c66bb6216e0667679fb424ee0bb67970 Mon Sep 17 00:00:00 2001 From: git-bruh Date: Thu, 25 Mar 2021 10:48:04 +0530 Subject: [PATCH] refactor (#5) * separate dir * lint * config * log unhandled exceptions * refactor * refactor webhook check * cogs cannnot be loaded from another dir * fix * fix * use logger.exception * Update README.md * Update README.md * fix * Update README.md * Create README.md * rm basedir * cogs * Update README.md * bold --- README.md | 55 ++--- bridge/README.md | 31 +++ main.py => bridge/bridge.py | 242 ++++++++++---------- {cogs => bridge/cogs}/.keep | 0 requirements.txt => bridge/requirements.txt | 2 +- {utils => misc}/migrate_emotes.py | 24 +- 6 files changed, 173 insertions(+), 181 deletions(-) create mode 100644 bridge/README.md rename main.py => bridge/bridge.py (74%) rename {cogs => bridge/cogs}/.keep (100%) rename requirements.txt => bridge/requirements.txt (64%) rename {utils => misc}/migrate_emotes.py (88%) diff --git a/README.md b/README.md index c06f2f1..9bd6490 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,22 @@ # matrix-discord-bridge -A simple non-puppeting bridge between Matrix and Discord written in Python. +A simple bridge between Matrix and Discord written in Python. -## Installation +This repository contains two bridges: +* A [puppeting appservice](appservice): The puppeting bridge written with minimal dependencies. Running this requires a self-hosted homeserver. -`pip install -r requirements.txt` +* A [non-puppeting bridge](bridge): The non-puppeting bridge written with `matrix-nio` and `discord.py`, most people would want to use this one. -## Usage - -* Run `main.py` to generate `config.json` - -* Edit `config.json` - -``` -{ - "homeserver": "https://matrix.org", - "username": "@name:matrix.org", - "password": "my-secret-password", - "token": "my-secret-token", - "discord_prefix": "my-command-prefix", # Prefix for Discord commands - "bridge": { - "channel_id": "room_id", # Bridge multiple channels and rooms - "channel_id2": "room_id2" - } -} -``` - -* Logs are saved to the `bot.log` file in `$PWD`. - -* Normal Discord bot functionality like commands can be added to the bot via [cogs](https://discordpy.readthedocs.io/en/latest/ext/commands/cogs.html), example [here](https://gist.github.com/EvieePy/d78c061a4798ae81be9825468fe146be). - -* Replace `guild.emojis` with `self.discord_client.emojis` (`Callbacks()`, `process_message()`) to make the Discord bot use emojis from ALL it's guilds. - -NOTE: [Privileged Intents](https://discordpy.readthedocs.io/en/latest/intents.html#privileged-intents) must be enabled for your Discord bot. - -## Screenshots -TODO +Check their READMEs for specific information. ## What Works -- [x] Sending messages -- [x] Discord webhooks (with avatars) -- [x] Attachments (Converted to URLs) -- [x] Typing status (Not very accurate) -- [x] Redacting messages -- [x] Editing messages +- [x] Puppeting (Appservice only, regular bridge only uses webhooks on Discord.) +- [x] Attachments (Converted to URLs.) +- [x] Typing Indicators (Per-user indicators on Appservice, otherwise sent as bot user.) +- [x] Message redaction - [x] Replies -- [x] Bridging multiple channels/rooms -- [x] `:emote:` in Matrix message converted to Discord emotes -- [x] Discord emotes bridged as inline images (Works on Element Web, Fluffychat) +- [x] Bridging multiple channels +- [x] Discord emojis displayed as inline images +- [x] Sending Discord emotes from Matrix (`:emote_name:`) +- [x] Mentioning Discord users via partial username (`@partialname`) diff --git a/bridge/README.md b/bridge/README.md new file mode 100644 index 0000000..2bbc106 --- /dev/null +++ b/bridge/README.md @@ -0,0 +1,31 @@ +## Installation + +`pip install -r requirements.txt` + +## Usage + +* Run `main.py` to generate `config.json` + +* Edit `config.json`: + +``` +{ + "homeserver": "https://matrix.org", + "username": "@name:matrix.org", + "password": "my-secret-password", # Matrix password. + "token": "my-secret-token", # Discord bot token. + "discord_cmd_prefix": "my-command-prefix", + "bridge": { + "channel_id": "room_id", + "channel_id2": "room_id2", # Bridge multiple rooms. + }, +} +``` + +This bridge does not use databases for keeping track of bridged rooms to avoid a dependency on persistent storage. This makes it easy to host on something like Heroku with the free tier. + +* Logs are saved to the `bridge.log` file in `$PWD`. + +* Normal Discord bot functionality like commands can be added to the bot via [cogs](https://discordpy.readthedocs.io/en/latest/ext/commands/cogs.html), example [here](https://gist.github.com/EvieePy/d78c061a4798ae81be9825468fe146be). + +NOTE: [Privileged Intents](https://discordpy.readthedocs.io/en/latest/intents.html#privileged-intents) must be enabled for your Discord bot. diff --git a/main.py b/bridge/bridge.py similarity index 74% rename from main.py rename to bridge/bridge.py index 396fd70..c77565d 100644 --- a/main.py +++ b/bridge/bridge.py @@ -3,9 +3,9 @@ import json import logging import os import re -import traceback import sys import uuid + import aiofiles import aiofiles.os import aiohttp @@ -20,8 +20,8 @@ def config_gen(config_file): "username": "@name:matrix.org", "password": "my-secret-password", "token": "my-secret-token", - "discord_prefix": "my-command-prefix", - "bridge": {"channel_id": "room_id"} + "discord_cmd_prefix": "my-command-prefix", + "bridge": {"channel_id": "room_id"}, } if not os.path.exists(config_file): @@ -37,7 +37,6 @@ def config_gen(config_file): config = config_gen("config.json") - message_store = {} @@ -56,17 +55,17 @@ class MatrixClient(nio.AsyncClient): self.add_callbacks() def start_discord(self): - # Disable everyone and role mentions. - allowed_mentions = discord.AllowedMentions(everyone=False, roles=False) - # Set command prefix for Discord bot. - command_prefix = config["discord_prefix"] # Intents to fetch members from guild. intents = discord.Intents.default() intents.members = True self.discord_client = DiscordClient( - self, allowed_mentions=allowed_mentions, - command_prefix=command_prefix, intents=intents + self, + allowed_mentions=discord.AllowedMentions( + everyone=False, roles=False + ), + command_prefix=config["discord_cmd_prefix"], + intents=intents, ) self.bg_task = self.loop.create_task( @@ -78,8 +77,7 @@ class MatrixClient(nio.AsyncClient): self.add_event_callback( callbacks.message_callback, - (nio.RoomMessageText, nio.RoomMessageMedia, - nio.RoomMessageEmote) + (nio.RoomMessageText, nio.RoomMessageMedia, nio.RoomMessageEmote), ) self.add_event_callback( @@ -107,16 +105,12 @@ class MatrixClient(nio.AsyncClient): await f.write(emote) async with aiofiles.open(emote_file, "rb") as f: - resp, maybe_keys = await self.upload( - f, content_type=content_type - ) + resp, maybe_keys = await self.upload(f, content_type=content_type) await aiofiles.os.remove(emote_file) if type(resp) != nio.UploadResponse: - self.logger.warning( - f"Failed to upload emote {emote_id}" - ) + self.logger.warning(f"Failed to upload emote {emote_id}") return self.uploaded_emotes[emote_id] = resp.content_uri @@ -125,18 +119,18 @@ class MatrixClient(nio.AsyncClient): async def get_fmt_body(self, body, emotes): replace_ = [ - # Code blocks - ("```", "
", "
"), - # Spoilers - ("||", "", ""), - # Strikethrough - ("~~", "", "") - ] + # Bold. + ("**", "", ""), + # Code blocks. + ("```", "
", "
"), + # Spoilers. + ("||", "", ""), + # Strikethrough. + ("~~", "", ""), + ] for replace in replace_: - for i in range(body.count(replace[0])): - i += 1 - + for i in range(1, body.count(replace[0]) + 1): if i % 2: body = body.replace(replace[0], replace[1], 1) else: @@ -147,77 +141,85 @@ class MatrixClient(nio.AsyncClient): if emote_: emote = f":{emote}:" body = body.replace( - emote, f"""\"{emote}\"""" + emote, + f"""\"{emote}\"""", ) return body - async def message_send(self, message, channel_id, emotes, - reply_id=None, edit_id=None): + async def message_send( + self, message, channel_id, emotes, reply_id=None, edit_id=None + ): room_id = config["bridge"][str(channel_id)] content = { "body": message, "format": "org.matrix.custom.html", "formatted_body": await self.get_fmt_body(message, emotes), - "msgtype": "m.text" + "msgtype": "m.text", } if reply_id: - reply_event = await self.room_get_event( - room_id, reply_id - ) + reply_event = await self.room_get_event(room_id, reply_id) reply_event = reply_event.event - content["m.relates_to"] = {"m.in_reply_to": {"event_id": reply_id}} - - content["formatted_body"] = f"""
\ + content = { + **content, + "m.relates_to": {"m.in_reply_to": {"event_id": reply_id}}, + "formatted_body": f"""
\ In reply to\ {reply_event.sender}\ -
{reply_event.body}
{content["formatted_body"]}""" - - if edit_id: - content["body"] = f" * {content['body']}" - - content["m.relates_to"] = { - "event_id": edit_id, "rel_type": "m.replace" +
{reply_event.body}
{content["formatted_body"]}""", } - content["m.new_content"] = { - "body": content["body"], - "formatted_body": content["formatted_body"], - "format": content["format"], - "msgtype": content["msgtype"] + if edit_id: + content = { + **content, + "body": f" * {content['body']}", + "formatted_body": f" * {content['formatted_body']}", + "m.relates_to": {"event_id": edit_id, "rel_type": "m.replace"}, + "m.new_content": {**content}, } message = await self.room_send( - room_id=room_id, - message_type="m.room.message", - content=content + room_id=room_id, message_type="m.room.message", content=content ) return message.event_id async def message_redact(self, message, channel_id): await self.room_redact( - room_id=config["bridge"][str(channel_id)], - event_id=message + room_id=config["bridge"][str(channel_id)], event_id=message ) - async def webhook_send(self, author, avatar, message, - event_id, channel_id, embed=None): + async def webhook_send( + self, author, avatar, message, event_id, channel_id, embed=None + ): channel = self.discord_client.channel_store[channel_id] - # Recreate hook if it was deleted. - hook = await self.discord_client.hook_create(channel) + hook_name = "matrix_bridge" + + hook = self.discord_client.webhook_cache.get(str(channel.id)) + + if not hook: + hooks = await channel.webhooks() + hook = discord.utils.get(hooks, name=hook_name) + + if not hook: + hook = await channel.create_webhook(name=hook_name) + + self.discord_client.webhook_cache[str(channel.id)] = hook # Username must be between 1 and 80 characters in length, # 'wait=True' allows us to store the sent message. try: hook = await hook.send( - username=author[:80], avatar_url=avatar, - content=message, embed=embed, wait=True + username=author[:80], + avatar_url=avatar, + content=message, + embed=embed, + wait=True, ) message_store[event_id] = hook @@ -232,7 +234,7 @@ class DiscordClient(discord.ext.commands.Bot): self.channel_store = {} - self.webhook_ids = set() + self.webhook_cache = {} self.ready = asyncio.Event() @@ -251,37 +253,20 @@ class DiscordClient(discord.ext.commands.Bot): cog = f"cogs.{cog[:-3]}" self.load_extension(cog) - async def hook_create(self, channel): - hook_name = "matrix_bridge" - - hooks = await channel.webhooks() - - # Check if webhook exists. - hook = discord.utils.get(hooks, name=hook_name) - if not hook: - hook = await channel.create_webhook(name=hook_name) - - self.webhook_ids.add(hook.id) - - return hook - async def to_return(self, channel_id, message=None): await self.matrix_client.ready.wait() - if str(channel_id) not in config["bridge"].keys(): - return True - - if message: - if message.webhook_id in self.webhook_ids: - return True + return str(channel_id) not in config["bridge"].keys() or ( + message + and message.webhook_id + in [hook.id for hook in self.webhook_cache.values()] + ) async def on_ready(self): for channel in config["bridge"].keys(): channel_ = self.get_channel(int(channel)) self.channel_store[channel] = channel_ - await self.hook_create(channel_) - self.ready.set() async def on_message(self, message): @@ -294,8 +279,10 @@ class DiscordClient(discord.ext.commands.Bot): content = await self.process_message(message) matrix_message = await self.matrix_client.message_send( - content[0], message.channel.id, - reply_id=content[1], emotes=content[2] + content[0], + message.channel.id, + reply_id=content[1], + emotes=content[2], ) message_store[message.id] = matrix_message @@ -309,8 +296,10 @@ class DiscordClient(discord.ext.commands.Bot): # Edit message only if it can be looked up in the cache. if before.id in message_store: await self.matrix_client.message_send( - content[0], after.channel.id, - edit_id=message_store[before.id], emotes=content[2] + content[0], + after.channel.id, + edit_id=message_store[before.id], + emotes=content[2], ) async def on_message_delete(self, message): @@ -371,8 +360,12 @@ class Callbacks(object): def get_channel(self, room): channel_id = next( - (channel_id for channel_id, room_id in config["bridge"].items() - if room_id == room.room_id), None + ( + channel_id + for channel_id, room_id in config["bridge"].items() + if room_id == room.room_id + ), + None, ) return channel_id @@ -380,10 +373,11 @@ class Callbacks(object): async def to_return(self, room, event): await self.matrix_client.discord_client.ready.wait() - if room.room_id not in config["bridge"].values() or \ - event.sender == self.matrix_client.user or \ - not self.matrix_client.listen: - return True + return ( + room.room_id not in config["bridge"].values() + or event.sender == self.matrix_client.user + or not self.matrix_client.listen + ) async def message_callback(self, room, event): message = event.body @@ -418,7 +412,8 @@ class Callbacks(object): await webhook_message.edit(content=edited_content) # Handle exception if edited message was deleted on Discord. except ( - discord.errors.NotFound, discord.errors.HTTPException + discord.errors.NotFound, + discord.errors.HTTPException, ) as e: self.matrix_client.logger.warning( f"Failed to edit message {edited_event}: {e}" @@ -429,9 +424,11 @@ class Callbacks(object): pass try: - if content_dict["m.relates_to"]["m.in_reply_to"]["event_id"] in \ - message_store.values(): - # Remove the first occurance of our bot's username if replying. + if ( + content_dict["m.relates_to"]["m.in_reply_to"]["event_id"] + in message_store.values() + ): + # Remove the first occurence of our bot's username if replying. # > <@discordbridge:something.org> [discord user] message = message.replace(f"<{config['username']}>", "", 1) except KeyError: @@ -489,13 +486,14 @@ class Callbacks(object): pass async def typing_callback(self, room, event): - if not room.typing_users \ - or room.room_id not in config["bridge"].values(): - return - - # Return if the event is sent by our bot. - if len(room.typing_users) == 1 and \ - self.matrix_client.user in room.typing_users: + if ( + not room.typing_users + or ( + len(room.typing_users) == 1 + and self.matrix_client.user in room.typing_users + ) + or room.room_id not in config["bridge"].values() + ): return # Get the corresponding Discord channel. @@ -506,8 +504,8 @@ class Callbacks(object): return async def process_message(self, message, channel_id): - mentions = re.findall(r"(^|\s)(@(\w*))", message) emotes = re.findall(r":(\w*):", message) + mentions = re.findall(r"(@(\w*))", message) # Get the guild from channel ID. guild = self.discord_client.channel_store[channel_id].guild @@ -522,15 +520,15 @@ class Callbacks(object): if emote_: message = message.replace(f":{emote}:", str(emote_)) - # mentions = [('', '@name', 'name'), (' ', '@', '')] + # mentions = [('@name', 'name'), ('@', '')] for mention in mentions: # Don't fetch member if mention is empty. # Single "@" without any name. - if mention[2]: - member = await guild.query_members(query=mention[2]) + if mention[1]: + member = await guild.query_members(query=mention[1]) if member: # Get first result. - message = message.replace(mention[1], member[0].mention) + message = message.replace(mention[0], member[0].mention) return message @@ -538,17 +536,17 @@ class Callbacks(object): async def main(): logging.basicConfig( level=logging.INFO, + format="%(asctime)s %(name)s:%(levelname)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", handlers=[ - logging.FileHandler("bot.log"), - logging.StreamHandler() - ] + logging.FileHandler("bridge.log"), + logging.StreamHandler(), + ], ) retry = 2 - matrix_client = MatrixClient( - config["homeserver"], config["username"] - ) + matrix_client = MatrixClient(config["homeserver"], config["username"]) while True: resp = await matrix_client.login(config["password"]) @@ -563,9 +561,7 @@ async def main(): try: await matrix_client.sync(full_state=True) except Exception: - matrix_client.logger.error( - f"Initial sync failed!\n{traceback.format_exc()}" - ) + matrix_client.logger.exception("Initial sync failed!") return False try: @@ -576,9 +572,8 @@ async def main(): await matrix_client.sync_forever(timeout=30000, full_state=True) except Exception: - matrix_client.logger.error( - f"Unknown exception occured\n{traceback.format_exc()}\n" - f"Retrying in {retry} seconds..." + matrix_client.logger.exception( + f"Unknown exception occured, retrying in {retry} seconds..." ) # Clear "ready" status. @@ -593,5 +588,6 @@ async def main(): await matrix_client.close() return False + if __name__ == "__main__": asyncio.run(main()) diff --git a/cogs/.keep b/bridge/cogs/.keep similarity index 100% rename from cogs/.keep rename to bridge/cogs/.keep diff --git a/requirements.txt b/bridge/requirements.txt similarity index 64% rename from requirements.txt rename to bridge/requirements.txt index a2a2d0e..320c9dc 100644 --- a/requirements.txt +++ b/bridge/requirements.txt @@ -1,3 +1,3 @@ aiofiles==0.6.0 discord.py==1.6.0 -matrix-nio==0.16.0 +matrix-nio==0.17.0 diff --git a/utils/migrate_emotes.py b/misc/migrate_emotes.py similarity index 88% rename from utils/migrate_emotes.py rename to misc/migrate_emotes.py index ff991ff..2f62c2d 100644 --- a/utils/migrate_emotes.py +++ b/misc/migrate_emotes.py @@ -17,7 +17,7 @@ def config_gen(config_file): "username": "@name:matrix.org", "password": "my-secret-password", "token": "my-secret-token", - "migrate": {"guild_id": "room_id"} + "migrate": {"guild_id": "room_id"}, } if not os.path.exists(config_file): @@ -66,16 +66,12 @@ class MatrixClient(nio.AsyncClient): await f.write(emote_) async with aiofiles.open(emote_file, "rb") as f: - resp, maybe_keys = await self.upload( - f, content_type=content_type - ) + resp, maybe_keys = await self.upload(f, content_type=content_type) await aiofiles.os.remove(emote_file) if type(resp) != nio.UploadResponse: - self.logger.warning( - f"Failed to upload {emote_name}" - ) + self.logger.warning(f"Failed to upload {emote_name}") return self.logger.info(f"Uploaded {emote_name}") @@ -101,9 +97,7 @@ class MatrixClient(nio.AsyncClient): resp = await self.room_put_state(room_id, event_type, content) if type(resp) == nio.RoomPutStateError: - self.logger.warning( - f"Failed to send emote state: {resp}" - ) + self.logger.warning(f"Failed to send emote state: {resp}") class DiscordClient(discord.Client): @@ -115,8 +109,8 @@ class DiscordClient(discord.Client): ) self.bg_task = self.loop.create_task( - self.log_exceptions(self.matrix_client) - ) + self.log_exceptions(self.matrix_client) + ) self.logger = logging.getLogger("discord_logger") @@ -138,9 +132,9 @@ class DiscordClient(discord.Client): f"Guild: {emote_guild.name} Room: {emote_room}" ) - await asyncio.gather(*map( - self.matrix_client.upload_emote, emote_guild.emojis - )) + await asyncio.gather( + *map(self.matrix_client.upload_emote, emote_guild.emojis) + ) self.logger.info("Sending state event to room...")