Merge remote-tracking branch 'tulir/master' into hs/portal-limit

.gitignore

@@ -14,4 +14,5 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
+*.pickle
 *.bak

.gitlab-ci.yml

@@ -14,17 +14,24 @@ build amd64:
   - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
   - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
+      apk add --update curl
+      rm -rf /var/cache/apk/*
+      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+    fi
 
 build arm64:
   stage: build
   tags:
   - arm64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
   - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
 
@@ -36,6 +43,6 @@ manifest:
   script:
   - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
   - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ $CI_COMMIT_BRANCH == "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ $CI_COMMIT_BRANCH != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

Dockerfile

@@ -1,19 +1,24 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.11
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
 
-RUN echo "@edge_main http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories
-RUN echo "@edge_testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
-RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories
+ARG TARGETARCH=amd64
+
+RUN echo $'\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
 
 RUN apk add --no-cache \
+      python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
-      py3-alembic@edge_testing \
+      py3-telethon-session-sqlalchemy@edge \
+      py3-alembic@edge \
       py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark@edge_testing \
+      py3-commonmark@edge \
       # Indirect dependencies
       py3-idna \
       #moviepy
@@ -22,33 +27,28 @@ RUN apk add --no-cache \
         py3-requests \
         #imageio
           py3-numpy \
-      #telethon
-        py3-rsa \
+      #py3-telethon@edge \ (outdated)
         # Optional for socks proxies
         py3-pysocks \
         # cryptg
           py3-cffi \
+	  py3-qrcode@edge \
       py3-brotli \
       # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
       netcat-openbsd \
-      # olm
-      olm-dev@edge_community \
-      # matrix-nio?
-      py3-future \
-      py3-atomicwrites \
-      py3-pycryptodome@edge_main \
-      py3-peewee@edge_community \
-      py3-pyrsistent@edge_community \
-      py3-jsonschema \
-      py3-aiofiles \
-      py3-cachetools@edge_community \
-      py3-prometheus-client@edge_community \
+      # encryption
+      olm-dev \
+      py3-pycryptodome \
       py3-unpaddedbase64 \
-      py3-pyaes@edge_testing \
-      py3-logbook@edge_testing
+      py3-future \
+      bash \
+      curl \
+      jq && \
+  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
+  chmod +x yq && mv yq /usr/bin/yq
 
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
@@ -64,7 +64,7 @@ RUN apk add --virtual .build-deps \
 COPY . /opt/mautrix-telegram
 RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
   # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && rm -rf mautrix_telegram
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
 
 VOLUME /data
 ENV UID=1337 GID=1337 \

README.md

@@ -7,14 +7,27 @@
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
+## Sponsors
+* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+### Wiki
+All setup and usage instructions are located in the GitHub
+[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+
+* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
+  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
+* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
+  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
+  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+
+### Features & Roadmap
+[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+contains a general overview of what is supported by the bridge.
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
 
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
+![Preview](preview.png)

ROADMAP.md

@@ -6,9 +6,9 @@
   * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
-  * [x] Typing notifications*
-  * [x] Read receipts*
-  * [x] Pinning messages*
+  * [x] Typing notifications
+  * [x] Read receipts
+  * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
@@ -28,7 +28,10 @@
 	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [ ] Message history
+  * [x] Message history
+    * [x] Manually (`!tg backfill`)
+    * [x] Automatically when creating portal
+    * [x] Automatically for missed messages
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -50,12 +53,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [x] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
-  * [ ] ‡ Secret chats (not yet supported by Telethon)
-  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
+  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
+  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
-\* Requires [double puppeting](https://github.com/tulir/mautrix-telegram/wiki/Authentication#replacing-telegram-accounts-matrix-puppet-with-matrix-account) to be enabled  
-† Information not automatically sent from source, i.e. implementation may not be possible  
+† Information not automatically sent from source, i.e. implementation may not be possible
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -21,7 +21,6 @@
 mxtg_config.load()
 config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
 
-
 AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
 
 # Interpret the config file for Python logging.
@@ -55,7 +54,8 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True)
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        render_as_batch=True)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -76,7 +76,8 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -0,0 +1,32 @@
+"""Store Matrix avatar URL in database
+
+Revision ID: 3e3745baa458
+Revises: dff56c93da8d
+Create Date: 2020-06-15 14:32:10.454033
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3e3745baa458'
+down_revision = 'dff56c93da8d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.drop_column('avatar_url')
+
+    # ### end Alembic commands ###

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -0,0 +1,30 @@
+"""Add double puppet base URL to puppet table
+
+Revision ID: 888275d58e57
+Revises: a328bf4f0932
+Create Date: 2020-10-14 18:52:00.730666
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '888275d58e57'
+down_revision = 'a328bf4f0932'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('base_url')
+    # ### end Alembic commands ###

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -0,0 +1,38 @@
+"""Store encryption state event in db
+
+Revision ID: a328bf4f0932
+Revises: ccbaff858240
+Create Date: 2020-07-11 21:31:27.059813
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from mautrix.client.state_store.sqlalchemy import SerializableType
+from mautrix.types import RoomEncryptionStateEventContent
+
+# revision identifiers, used by Alembic.
+revision = 'a328bf4f0932'
+down_revision = 'ccbaff858240'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encryption',
+                                      SerializableType(RoomEncryptionStateEventContent),
+                                      nullable=True))
+        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.drop_column('is_encrypted')
+        batch_op.drop_column('has_full_member_list')
+        batch_op.drop_column('encryption')
+    # ### end Alembic commands ###