Matthias/conduit
1
0
Fork 0
Code Issues 179 Pull requests 44 Projects Releases 5 Packages Wiki Activity

Merge branch 'next' into 'master'

Browse source 
Release 0.3

See merge request famedly/conduit!288
This commit is contained in:
Timo Kösters 2022-02-04 17:37:26 +00:00
commit 9c8c784fe7
79 changed files with 5889 additions and 4132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dockerignore
View file

@ -14,6 +14,8 @@ docker-compose*
# Git folder
.git
.gitea
.gitlab
.github
# Dot files
.env

410
.gitlab-ci.yml
View file

@ -8,7 +8,10 @@ variables:
GIT_SUBMODULE_STRATEGY: recursive
FF_USE_FASTZIP: 1
CACHE_COMPRESSION_LEVEL: fastest
# Docker in Docker
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
DOCKER_DRIVER: overlay2
# --------------------------------------------------------------------- #
# Cargo: Compiling for different architectures #
@ -20,265 +23,202 @@ variables:
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG'
- if: "$CI_COMMIT_TAG"
- if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set.
interruptible: true
image: "rust:latest"
image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
tags: ["docker"]
cache:
paths:
- cargohome
- target/
key: "build_cache-$TARGET-release"
services: ["docker:dind"]
variables:
CARGO_PROFILE_RELEASE_LTO=true
CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
SHARED_PATH: $CI_PROJECT_DIR/shared
CARGO_PROFILE_RELEASE_LTO: "true"
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
before_script:
- 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- 'apt-get update -yqq'
- 'echo "Installing packages: $NEEDED_PACKAGES"'
- "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
- "rustup target add $TARGET"
- "rustup show && rustc --version && cargo --version" # Print version info for debugging
# fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
- "mkdir -p $SHARED_PATH/cargo"
- "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo"
- "cp -r $RUSTUP_HOME $SHARED_PATH"
- "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup"
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
- if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi
script:
- time cargo build --target $TARGET --release
# cross-compile conduit for target
- 'time cross build --target="$TARGET" --locked --release'
- 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
# print information about linking for debugging
- "file conduit-$TARGET" # print file information
- 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
cache:
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
key: "cargo-cache-$TARGET"
paths:
- $SHARED_PATH/cargo/registry/index
- $SHARED_PATH/cargo/registry/cache
- $SHARED_PATH/cargo/git/db
artifacts:
expire_in: never
build:release:cargo:x86_64-unknown-linux-gnu:
build:release:cargo:x86_64-unknown-linux-musl-with-debug:
extends: .build-cargo-shared-settings
variables:
TARGET: "x86_64-unknown-linux-gnu"
CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
TARGET: "x86_64-unknown-linux-musl"
after_script:
- "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug"
artifacts:
name: "conduit-x86_64-unknown-linux-gnu"
name: "conduit-x86_64-unknown-linux-musl-with-debug"
paths:
- "conduit-x86_64-unknown-linux-gnu"
expose_as: "Conduit for x86_64-unknown-linux-gnu"
build:release:cargo:armv7-unknown-linux-gnueabihf:
extends: .build-cargo-shared-settings
variables:
TARGET: "armv7-unknown-linux-gnueabihf"
NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross"
CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc
CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++
artifacts:
name: "conduit-armv7-unknown-linux-gnueabihf"
paths:
- "conduit-armv7-unknown-linux-gnueabihf"
expose_as: "Conduit for armv7-unknown-linux-gnueabihf"
build:release:cargo:aarch64-unknown-linux-gnu:
extends: .build-cargo-shared-settings
variables:
TARGET: "aarch64-unknown-linux-gnu"
NEEDED_PACKAGES: "build-essential gcc-8-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross"
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-8"
TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-8"
artifacts:
name: "conduit-aarch64-unknown-linux-gnu"
paths:
- "conduit-aarch64-unknown-linux-gnu"
expose_as: "Conduit for aarch64-unknown-linux-gnu"
- "conduit-x86_64-unknown-linux-musl-with-debug"
expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug"
build:release:cargo:x86_64-unknown-linux-musl:
extends: .build-cargo-shared-settings
image: "rust:alpine"
variables:
TARGET: "x86_64-unknown-linux-musl"
before_script:
- 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- "rustup target add $TARGET"
- "apk add libc-dev"
artifacts:
name: "conduit-x86_64-unknown-linux-musl"
paths:
- "conduit-x86_64-unknown-linux-musl"
expose_as: "Conduit for x86_64-unknown-linux-musl"
build:release:cargo:arm-unknown-linux-musleabihf:
extends: .build-cargo-shared-settings
variables:
TARGET: "arm-unknown-linux-musleabihf"
artifacts:
name: "conduit-arm-unknown-linux-musleabihf"
paths:
- "conduit-arm-unknown-linux-musleabihf"
expose_as: "Conduit for arm-unknown-linux-musleabihf"
build:release:cargo:armv7-unknown-linux-musleabihf:
extends: .build-cargo-shared-settings
variables:
TARGET: "armv7-unknown-linux-musleabihf"
artifacts:
name: "conduit-armv7-unknown-linux-musleabihf"
paths:
- "conduit-armv7-unknown-linux-musleabihf"
expose_as: "Conduit for armv7-unknown-linux-musleabihf"
build:release:cargo:aarch64-unknown-linux-musl:
extends: .build-cargo-shared-settings
variables:
TARGET: "aarch64-unknown-linux-musl"
artifacts:
name: "conduit-aarch64-unknown-linux-musl"
paths:
- "conduit-aarch64-unknown-linux-musl"
expose_as: "Conduit for aarch64-unknown-linux-musl"
.cargo-debug-shared-settings:
extends: ".build-cargo-shared-settings"
rules:
- if: '$CI_COMMIT_BRANCH'
- if: '$CI_COMMIT_TAG'
- if: '$CI_COMMIT_BRANCH != "master"'
cache:
key: "build_cache-$TARGET-debug"
key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
script:
- "time cargo build --target $TARGET"
# cross-compile conduit for target
- 'time time cross build --target="$TARGET" --locked'
- 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
# print information about linking for debugging
- "file conduit-debug-$TARGET" # print file information
- 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
artifacts:
expire_in: 4 weeks
build:debug:cargo:x86_64-unknown-linux-gnu:
extends: ".cargo-debug-shared-settings"
variables:
TARGET: "x86_64-unknown-linux-gnu"
artifacts:
name: "conduit-debug-x86_64-unknown-linux-gnu"
paths:
- "conduit-debug-x86_64-unknown-linux-gnu"
expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"
build:debug:cargo:x86_64-unknown-linux-musl:
extends: ".cargo-debug-shared-settings"
image: "rust:alpine"
variables:
TARGET: "x86_64-unknown-linux-musl"
before_script:
- 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- "rustup target add $TARGET"
- "apk add libc-dev"
artifacts:
name: "conduit-debug-x86_64-unknown-linux-musl"
paths:
- "conduit-debug-x86_64-unknown-linux-musl"
expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
# --------------------------------------------------------------------- #
# Cargo: Compiling deb packages for different architectures #
# --------------------------------------------------------------------- #
.build-cargo-deb-shared-settings:
stage: "build"
needs: [ ]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG'
interruptible: true
image: "rust:latest"
tags: ["docker"]
cache:
paths:
- cargohome
- target/
key: "build_cache-deb-$TARGET"
before_script:
- 'echo "Building debian package for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- 'apt-get update -yqq'
- 'echo "Installing packages: $NEEDED_PACKAGES"'
- "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
- "rustup target add $TARGET"
- "cargo install cargo-deb"
script:
- time cargo deb --target $TARGET
- 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"'
build:cargo-deb:x86_64-unknown-linux-gnu:
extends: .build-cargo-deb-shared-settings
variables:
TARGET: "x86_64-unknown-linux-gnu"
NEEDED_PACKAGES: ""
artifacts:
name: "conduit-x86_64-unknown-linux-gnu.deb"
paths:
- "conduit-x86_64-unknown-linux-gnu.deb"
expose_as: "Debian Package x86_64"
# --------------------------------------------------------------------- #
# Create and publish docker image #
# --------------------------------------------------------------------- #
# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
.docker-shared-settings:
stage: "build docker image"
needs: []
interruptible: true
image:
name: "gcr.io/kaniko-project/executor:debug"
entrypoint: [""]
image: jdrouet/docker-with-buildx:stable
tags: ["docker"]
variables:
# Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
before_script:
- "mkdir -p /kaniko/.docker"
- 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
build:docker:next:
extends: .docker-shared-settings
services:
- docker:dind
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
- "build:release:cargo:arm-unknown-linux-musleabihf"
- "build:release:cargo:armv7-unknown-linux-musleabihf"
- "build:release:cargo:aarch64-unknown-linux-musl"
variables:
PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
cache:
paths:
- docker_cache
key: "$CI_JOB_NAME"
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
# Only log in to Dockerhub if the credentials are given:
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
script:
# Prepare buildx to build multiarch stuff:
- docker context create 'ci-context'
- docker buildx create --name 'multiarch-builder' --use 'ci-context'
# Copy binaries to their docker arch path
- mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
- mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
- mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
- mv ./conduit-aarch64-unknown-linux-musl linux/arm64
- 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
# Build and push image:
- >
/kaniko/executor
$KANIKO_CACHE_ARGS
--context $CI_PROJECT_DIR
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
docker buildx build
--pull
--push
--cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
--cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
--build-arg CREATED=$CREATED
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
--destination "$CI_REGISTRY_IMAGE/conduit:next"
--destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
--destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
--platform "$PLATFORMS"
--tag "$TAG"
--tag "$TAG-alpine"
--tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
--file "$DOCKER_FILE" .
docker:next:gitlab:
extends: .docker-shared-settings
rules:
- if: '$CI_COMMIT_BRANCH == "next"'
variables:
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
build:docker:master:
docker:next:dockerhub:
extends: .docker-shared-settings
rules:
- if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
variables:
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
docker:master:gitlab:
extends: .docker-shared-settings
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
script:
- >
/kaniko/executor
$KANIKO_CACHE_ARGS
--context $CI_PROJECT_DIR
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
--destination "$CI_REGISTRY_IMAGE/conduit:latest"
--destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine"
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
variables:
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
build:docker:tags:
docker:master:dockerhub:
extends: .docker-shared-settings
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
script:
- >
/kaniko/executor
$KANIKO_CACHE_ARGS
--context $CI_PROJECT_DIR
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
--destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG"
--destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine"
rules:
- if: '$CI_COMMIT_TAG'
- if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
variables:
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
# --------------------------------------------------------------------- #
# Run tests #
@ -286,27 +226,26 @@ build:docker:tags:
test:cargo:
stage: "test"
needs: [ ]
image: "rust:latest"
tags: [ "docker" ]
needs: []
image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
tags: ["docker"]
variables:
CARGO_HOME: "cargohome"
cache:
paths:
- target
- cargohome
key: test_cache
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
interruptible: true
before_script:
- mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
- apt-get update -yqq
- apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
- rustup component add clippy rustfmt
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
- if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
script:
- rustc --version && cargo --version # Print version info for debugging
- rustc --version && cargo --version # Print version info for debugging
- cargo fmt --all -- --check
- cargo test --workspace --verbose --locked
- cargo clippy
- "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
- "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
artifacts:
when: always
reports:
junit: report.xml
codequality: gl-code-quality-report.json
test:sytest:
stage: "test"
@ -315,8 +254,8 @@ test:sytest:
- "build:debug:cargo:x86_64-unknown-linux-musl"
image:
name: "valkum/sytest-conduit:latest"
entrypoint: [ "" ]
tags: [ "docker" ]
entrypoint: [""]
tags: ["docker"]
variables:
PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
before_script:
@ -329,7 +268,7 @@ test:sytest:
script:
- "SYTEST_EXIT_CODE=0"
- "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
- "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
- 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap'
- "exit $SYTEST_EXIT_CODE"
artifacts:
when: always
@ -339,31 +278,6 @@ test:sytest:
reports:
junit: "$CI_PROJECT_DIR/sytest.xml"
test:register:element-web-stable:
stage: "test"
needs:
- "build:debug:cargo:x86_64-unknown-linux-gnu"
image: "buildkite/puppeteer:latest"
tags: [ "docker" ]
interruptible: true
script:
- "CONDUIT_CONFIG=tests/test-config.toml ./conduit-debug-x86_64-unknown-linux-gnu > conduit.log &"
- "cd tests/client-element-web/"
- "npm install puppeteer"
- "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
- "killall --regexp \"conduit\""
- "cd ../.."
- "cat conduit.log"
artifacts:
paths:
- "tests/client-element-web/*.png"
- "*.log"
expire_in: 1 week
when: always
retry: 1
# --------------------------------------------------------------------- #
# Store binaries as package so they have download urls #
# --------------------------------------------------------------------- #
@ -371,25 +285,31 @@ test:register:element-web-stable:
publish:package:
stage: "upload artifacts"
needs:
- "build:release:cargo:x86_64-unknown-linux-gnu"
- "build:release:cargo:armv7-unknown-linux-gnueabihf"
- "build:release:cargo:aarch64-unknown-linux-gnu"
- "build:release:cargo:x86_64-unknown-linux-musl"
- "build:cargo-deb:x86_64-unknown-linux-gnu"
- "build:release:cargo:arm-unknown-linux-musleabihf"
- "build:release:cargo:armv7-unknown-linux-musleabihf"
- "build:release:cargo:aarch64-unknown-linux-musl"
# - "build:cargo-deb:x86_64-unknown-linux-gnu"
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG'
- if: "$CI_COMMIT_TAG"
image: curlimages/curl:latest
tags: ["docker"]
variables:
GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
script:
- 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"'
# Avoid duplicate pipelines
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
- if: "$CI_COMMIT_BRANCH"

35
APPSERVICES.md
View file

@ -2,16 +2,9 @@
## Getting help
If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in [#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org) or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
## Tested appservices
Here are some appservices we tested and that work with Conduit:
- matrix-appservice-discord
- mautrix-hangouts
- mautrix-telegram
## Set up the appservice
## Set up the appservice - general instructions
Follow whatever instructions are given by the appservice. This usually includes
downloading, changing its config (setting domain, homeserver url, port etc.)
@ -25,7 +18,7 @@ First, go into the #admins room of your homeserver. The first person that
registered on the homeserver automatically joins it. Then send a message into
the room like this:
@conduit:your.server.name: register_appservice
@conduit:your.server.name: register-appservice
```
paste
the
@ -38,7 +31,7 @@ the room like this:
```
You can confirm it worked by sending a message like this:
`@conduit:your.server.name: list_appservices`
`@conduit:your.server.name: list-appservices`
The @conduit bot should answer with `Appservices (1): your-bridge`
@ -46,3 +39,23 @@ Then you are done. Conduit will send messages to the appservices and the
appservice can send requests to the homeserver. You don't need to restart
Conduit, but if it doesn't work, restarting while the appservice is running
could help.
## Appservice-specific instructions
### Remove an appservice
To remove an appservice go to your admin room and execute
`@conduit:your.server.name: unregister-appservice <name>`
where `<name>` one of the output of `list-appservices`.
### Tested appservices
These appservices have been tested and work with Conduit without any extra steps:
- [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord)
- [mautrix-hangouts](https://github.com/mautrix/hangouts/)
- [mautrix-telegram](https://github.com/mautrix/telegram/)
- [mautrix-signal](https://github.com/mautrix/signal/) from version `0.2.2` forward.
- [heisenbridge](https://github.com/hifi/heisenbridge/)

11
CROSS_COMPILE.md
View file

@ -1,11 +0,0 @@
Install docker:
```
$ sudo apt install docker
$ sudo usermod -aG docker $USER
$ exec sudo su -l $USER
$ sudo systemctl start docker
$ cargo install cross
$ cross build --release --target armv7-unknown-linux-musleabihf
```
The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit

1152
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

43
Cargo.toml
View file

@ -6,8 +6,9 @@ authors = ["timokoesters <timo@koesters.xyz>"]
homepage = "https://conduit.rs"
repository = "https://gitlab.com/famedly/conduit"
readme = "README.md"
version = "0.2.0"
edition = "2018"
version = "0.3.0"
rust-version = "1.56"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -18,8 +19,8 @@ edition = "2018"
rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests
# Used for matrix spec type definitions and helpers
ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
ruma = { git = "https://github.com/ruma/ruma", rev = "f7a10a7e471b59d3096be2695c2a05d407d80df1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
@ -28,6 +29,7 @@ tokio = "1.11.0"
# Used for storing data permanently
sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
#sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
persy = { version = "1.2" , optional = true, features=["background_ops"] }
# Used for the http request / response body type for Ruma endpoints used with reqwest
bytes = "1.1.0"
@ -36,21 +38,17 @@ http = "0.2.4"
# Used to find data directory for default db path
directories = "3.0.2"
# Used for ruma wrapper
serde_json = { version = "1.0.67", features = ["raw_value"] }
serde_json = { version = "1.0.70", features = ["raw_value"] }
# Used for appservice registration files
serde_yaml = "0.8.20"
# Used for pdu definition
serde = "1.0.130"
serde = { version = "1.0.130", features = ["rc"] }
# Used for secure identifiers
rand = "0.8.4"
# Used to hash passwords
rust-argon2 = "0.8.3"
# Used to send requests
reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls-native-roots", "socks"] }
# Custom TLS verifier
rustls = { version = "0.19.1", features = ["dangerous_configuration"] }
rustls-native-certs = "0.5.0"
webpki = "0.22.0"
reqwest = { default-features = false, features = ["rustls-tls", "socks"], git = "https://github.com/timokoesters/reqwest", rev = "57b7cf4feb921573dfafad7d34b9ac6e44ead0bd" }
# Used for conduit::Error type
thiserror = "1.0.28"
# Used to generate thumbnails for images
@ -68,11 +66,9 @@ jsonwebtoken = "7.2.0"
# Performance measurements
tracing = { version = "0.1.26", features = ["release_max_level_warn"] }
tracing-subscriber = "0.2.20"
tracing-opentelemetry = "0.15.0"
tracing-flame = "0.1.0"
opentelemetry = { version = "0.16.0", features = ["rt-tokio"] }
opentelemetry-jaeger = { version = "0.15.0", features = ["rt-tokio"] }
pretty_env_logger = "0.4.0"
lru-cache = "0.1.2"
rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] }
parking_lot = { version = "0.11.2", optional = true }
@ -80,14 +76,28 @@ crossbeam = { version = "0.8.1", optional = true }
num_cpus = "1.13.0"
threadpool = "1.8.1"
heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true }
thread_local = "1.1.3"
# used for TURN server authentication
hmac = "0.11.0"
sha-1 = "0.9.8"
# used for conduit's CLI and admin room command parsing
clap = { version = "3.0.10", default-features = false, features = ["std", "derive"] }
maplit = "1.0.2"
tikv-jemalloc-ctl = { version = "0.4.2", features = ["use_std"], optional = true }
tikv-jemallocator = { version = "0.4.1", features = ["unprefixed_malloc_on_supported_platforms"], optional = true }
[features]
default = ["conduit_bin", "backend_sqlite"]
default = ["conduit_bin", "backend_sqlite", "backend_rocksdb", "jemalloc"]
backend_sled = ["sled"]
backend_persy = ["persy", "parking_lot"]
backend_sqlite = ["sqlite"]
backend_heed = ["heed", "crossbeam"]
sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]
backend_rocksdb = ["rocksdb"]
jemalloc = ["tikv-jemalloc-ctl", "tikv-jemallocator"]
sqlite = ["rusqlite", "parking_lot", "tokio/signal"]
conduit_bin = [] # TODO: add rocket to this when it is optional
[[bin]]
@ -122,13 +132,12 @@ maintainer-scripts = "debian/"
systemd-units = { unit-name = "matrix-conduit" }
[profile.dev]
lto = 'thin'
lto = 'off'
incremental = true
[profile.release]
lto = 'thin'
incremental = true
codegen-units=32
# If you want to make flamegraphs, enable debug info:
# debug = true

23
Cross.toml Normal file
View file

@ -0,0 +1,23 @@
[build.env]
# CI uses an S3 endpoint to store sccache artifacts, so their config needs to
# be available in the cross container as well
passthrough = [
"RUSTC_WRAPPER",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"SCCACHE_BUCKET",
"SCCACHE_ENDPOINT",
"SCCACHE_S3_USE_SSL",
]
[target.aarch64-unknown-linux-musl]
image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest"
[target.arm-unknown-linux-musleabihf]
image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest"
[target.armv7-unknown-linux-musleabihf]
image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest"
[target.x86_64-unknown-linux-musl]
image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest"

77
DEPLOY.md
View file

@ -1,46 +1,56 @@
# Deploying Conduit
## Getting help
If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
> ## Getting help
>
> If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us
> in `#conduit:fachschaften.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
## Installing Conduit
Although you might be able to compile Conduit for Windows, we do recommend running it on a linux server. We therefore
only offer Linux binaries.
You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
| CPU Architecture | GNU (Ubuntu, Debian, ArchLinux, ...) | MUSL (Alpine, ... ) |
| -------------------- | ------------------------------------- | ----------------------- |
| x84_64 / amd64 | [Download][x84_64-gnu] | [Download][x84_64-musl] |
| armv7 (Raspberry Pi) | [Download][armv7-gnu] | - |
| armv8 / aarch64 | [Download][armv8-gnu] | - |
| CPU Architecture | Download stable version | Download development version |
| ------------------------------------------- | ------------------------------ | ---------------------------- |
| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] | [Download][x84_64-musl-next] |
| armv6 | [Download][armv6-musl-master] | [Download][armv6-musl-next] |
| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl-master] | [Download][armv7-musl-next] |
| armv8 / aarch64 | [Download][armv8-musl-master] | [Download][armv8-musl-next] |
[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu
[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf
[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu
[x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
[armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
[armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
[armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
[x84_64-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
[armv6-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
[armv7-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
[armv8-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
```bash
$ sudo wget -O /usr/local/bin/matrix-conduit <url>
$ sudo chmod +x /usr/local/bin/matrix-conduit
```
Alternatively, you may compile the binary yourself using
Alternatively, you may compile the binary yourself
```bash
$ sudo apt install libclang-dev build-essential
```
```bash
$ cargo build --release
```
Note that this currently requires Rust 1.50.
If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md).
## Adding a Conduit user
While Conduit can run as any user it is usually better to use dedicated users for different services.
This also allows you to make sure that the file permissions are correctly set up.
While Conduit can run as any user it is usually better to use dedicated users for different services. This also allows
you to make sure that the file permissions are correctly set up.
In Debian you can use this command to create a Conduit user:
@ -50,9 +60,8 @@ sudo adduser --system conduit --no-create-home
## Setting up a systemd service
Now we'll set up a systemd service for Conduit, so it's easy to start/stop
Conduit and set it to autostart when your server reboots. Simply paste the
default systemd service you can find below into
Now we'll set up a systemd service for Conduit, so it's easy to start/stop Conduit and set it to autostart when your
server reboots. Simply paste the default systemd service you can find below into
`/etc/systemd/system/conduit.service`.
```systemd
@ -77,10 +86,10 @@ Finally, run
$ sudo systemctl daemon-reload
```
## Creating the Conduit configuration file
Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.**
Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment
to read it. You need to change at least the server name.**
```toml
[global]
@ -128,8 +137,8 @@ address = "127.0.0.1" # This makes sure Conduit can only be reached using the re
## Setting the correct file permissions
As we are using a Conduit specific user we need to allow it to read the config.
To do that you can run this command on Debian:
As we are using a Conduit specific user we need to allow it to read the config. To do that you can run this command on
Debian:
```bash
sudo chown -R conduit:nogroup /etc/matrix-conduit
@ -142,7 +151,6 @@ sudo mkdir -p /var/lib/matrix-conduit/conduit_db
sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
```
## Setting up the Reverse Proxy
This depends on whether you use Apache, Nginx or another web server.
@ -162,9 +170,6 @@ AllowEncodedSlashes NoDecode
ProxyPass /_matrix/ http://127.0.0.1:6167/_matrix/ nocanon
ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/
Include /etc/letsencrypt/options-ssl-apache.conf
SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS
SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS
</VirtualHost>
```
@ -174,11 +179,9 @@ SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT
$ sudo systemctl reload apache2
```
### Nginx
If you use Nginx and not Apache, add the following server section inside the
http section of `/etc/nginx/nginx.conf`
If you use Nginx and not Apache, add the following server section inside the http section of `/etc/nginx/nginx.conf`
```nginx
server {
@ -201,13 +204,13 @@ server {
include /etc/letsencrypt/options-ssl-nginx.conf;
}
```
**You need to make some edits again.** When you are done, run
```bash
$ sudo systemctl reload nginx
```
## SSL Certificate
The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
@ -216,7 +219,6 @@ The easiest way to get an SSL certificate, if you don't have one already, is to
$ sudo certbot -d your.server.name
```
## You're done!
Now you can start Conduit with:
@ -242,4 +244,5 @@ $ curl https://your.server.name/_matrix/client/versions
$ curl https://your.server.name:8448/_matrix/client/versions
```
If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md).
- To check if your server can talk with other homeservers, you can use the [Matrix Federation Tester](https://federationtester.matrix.org/)
- If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md).

147
Dockerfile
View file

@ -1,97 +1,80 @@
# Using multistage build:
# https://docs.docker.com/develop/develop-images/multistage-build/
# https://whitfin.io/speeding-up-rust-docker-builds/
# syntax=docker/dockerfile:1
FROM docker.io/rust:1.58-bullseye AS builder
WORKDIR /usr/src/conduit
# Install required packages to build Conduit and it's dependencies
RUN apt update && apt -y install libclang-dev
########################## BUILD IMAGE ##########################
# Alpine build image to build Conduit's statically compiled binary
FROM alpine:3.14 as builder
# == Build dependencies without our own code separately for caching ==
#
# Need a fake main.rs since Cargo refuses to build anything otherwise.
#
# See https://github.com/rust-lang/cargo/issues/2644 for a Cargo feature
# request that would allow just dependencies to be compiled, presumably
# regardless of whether source files are available.
RUN mkdir src && touch src/lib.rs && echo 'fn main() {}' > src/main.rs
COPY Cargo.toml Cargo.lock ./
RUN cargo build --release && rm -r src
# Install packages needed for building all crates
RUN apk add --no-cache \
cargo \
openssl-dev
# Copy over actual Conduit sources
COPY src src
# Specifies if the local project is build or if Conduit gets build
# from the official git repository. Defaults to the git repo.
ARG LOCAL=false
# Specifies which revision/commit is build. Defaults to HEAD
ARG GIT_REF=origin/master
# main.rs and lib.rs need their timestamp updated for this to work correctly since
# otherwise the build with the fake main.rs from above is newer than the
# source files (COPY preserves timestamps).
#
# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
RUN touch src/main.rs && touch src/lib.rs && cargo build --release
# Copy project files from current folder
COPY . .
# Build it from the copied local files or from the official git repository
RUN if [[ $LOCAL == "true" ]]; then \
mv ./docker/healthcheck.sh . ; \
echo "Building from local source..." ; \
cargo install --path . ; \
else \
echo "Building revision '${GIT_REF}' from online source..." ; \
cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF} ; \
echo "Loadings healthcheck script from online source..." ; \
wget "https://gitlab.com/famedly/conduit/-/raw/${GIT_REF#origin/}/docker/healthcheck.sh" ; \
fi
# ---------------------------------------------------------------------------------------------------------------
# Stuff below this line actually ends up in the resulting docker image
# ---------------------------------------------------------------------------------------------------------------
FROM docker.io/debian:bullseye-slim AS runner
########################## RUNTIME IMAGE ##########################
# Create new stage with a minimal image for the actual
# runtime image/container
FROM alpine:3.14
ARG CREATED
ARG VERSION
ARG GIT_REF=origin/master
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
# including a custom label specifying the build command
LABEL org.opencontainers.image.created=${CREATED} \
org.opencontainers.image.authors="Conduit Contributors" \
org.opencontainers.image.title="Conduit" \
org.opencontainers.image.version=${VERSION} \
org.opencontainers.image.vendor="Conduit Contributors" \
org.opencontainers.image.description="A Matrix homeserver written in Rust" \
org.opencontainers.image.url="https://conduit.rs/" \
org.opencontainers.image.revision=${GIT_REF} \
org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.documentation="" \
org.opencontainers.image.ref.name="" \
org.label-schema.docker.build="docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)" \
maintainer="Weasy666"
# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
# Standard port on which Conduit launches.
# You still need to map the port when using the docker command or docker-compose.
EXPOSE 6167
# Copy config files from context and the binary from
# the "builder" stage to the current stage into folder
# /srv/conduit and create data folder for database
# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs.
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
CONDUIT_PORT=6167
# Conduit needs:
# ca-certificates: for https
# iproute2 & wget: for the healthcheck script
RUN apt update && apt -y install \
ca-certificates \
iproute2 \
wget \
&& rm -rf /var/lib/apt/lists/*
# Created directory for the database and media files
RUN mkdir -p /srv/conduit/.local/share/conduit
COPY --from=builder /root/.cargo/bin/conduit /srv/conduit/
COPY --from=builder ./healthcheck.sh /srv/conduit/
# Add www-data user and group with UID 82, as used by alpine
# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
RUN set -x ; \
addgroup -Sg 82 www-data 2>/dev/null ; \
adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \
addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1
# Change ownership of Conduit files to www-data user and group
RUN chown -cR www-data:www-data /srv/conduit
# Install packages needed to run Conduit
RUN apk add --no-cache \
ca-certificates \
curl \
libgcc
# Test if Conduit is still alive, uses the same endpoint as Element
HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
# Set user to www-data
USER www-data
# Copy over the actual Conduit binary from the builder stage
COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit
# Improve security: Don't run stuff as root, that does not need to run as root
# Add 'conduit' user and group (100:82). The UID:GID choice is to be compatible
# with previous, Alpine-based containers, where the user and group were both
# named 'www-data'.
RUN set -x ; \
groupadd -r -g 82 conduit ; \
useradd -r -M -d /srv/conduit -o -u 100 -g conduit conduit && exit 0 ; exit 1
# Change ownership of Conduit files to conduit user and group and make the healthcheck executable:
RUN chown -cR conduit:conduit /srv/conduit && \
chmod +x /srv/conduit/healthcheck.sh
# Change user to conduit, no root permissions afterwards:
USER conduit
# Set container home directory
WORKDIR /srv/conduit
# Run Conduit
# Run Conduit and print backtraces on panics
ENV RUST_BACKTRACE=1
ENTRYPOINT [ "/srv/conduit/conduit" ]

18
README.md
View file

@ -1,4 +1,5 @@
# Conduit
### A Matrix homeserver written in Rust
#### What is the goal?
@ -7,7 +8,6 @@ An efficient Matrix homeserver that's easy to set up and just works. You can ins
it on a mini-computer like the Raspberry Pi to host Matrix for your family,
friends or company.
#### Can I try it out?
Yes! You can test our Conduit instance by opening a Matrix client (<https://app.element.io> or Element Android for
@ -17,7 +17,6 @@ It is hosted on a ODROID HC 2 with 2GB RAM and a SAMSUNG Exynos 5422 CPU, which
was used in the Samsung Galaxy S5. It joined many big rooms including Matrix
HQ.
#### What is the current status?
As of 2021-09-01, Conduit is Beta, meaning you can join and participate in most
@ -31,26 +30,23 @@ There are still a few important features missing:
Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3).
#### How can I deploy my own?
Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md)\
Debian package: [debian/README.Debian](debian/README.Debian)\
Docker: [docker/README.md](docker/README.md)
- Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md)
- Debian package: [debian/README.Debian](debian/README.Debian)
- Docker: [docker/README.md](docker/README.md)
If you want to connect an Appservice to Conduit, take a look at [APPSERVICES.md](APPSERVICES.md).
#### How can I contribute?
1. Look for an issue you would like to work on and make sure it's not assigned
to other users
2. Ask someone to assign the issue to you (comment on the issue or chat in
#conduit:nordgedanken.dev)
3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
[#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org))
3. Fork the repo and work on the issue.[#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org) is happy to help :)
4. Submit a MR
#### Thanks to
Thanks to Famedly, Prototype Fund (DLR and German BMBF) and all other individuals for financially supporting this project.
@ -60,13 +56,11 @@ Thanks to the contributors to Conduit and all libraries we use, for example:
- Ruma: A clean library for the Matrix Spec in Rust
- Rocket: A flexible web framework
#### Donate
Liberapay: <https://liberapay.com/timokoesters/>\
Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n`
#### Logo
Lightning Bolt Logo: https://github.com/mozilla/fxemoji/blob/gh-pages/svgs/nature/u26A1-bolt.svg \

17
conduit-example.toml
View file

@ -1,11 +1,15 @@
[global]
# The server_name is the name of this server. It is used as a suffix for user
# The server_name is the pretty name of this server. It is used as a suffix for user
# and room ids. Examples: matrix.org, conduit.rs
# The Conduit server needs to be reachable at https://your.server.name/ on port
# 443 (client-server) and 8448 (federation) OR you can create /.well-known
# files to redirect requests. See
# The Conduit server needs all /_matrix/ requests to be reachable at
# https://your.server.name/ on port 443 (client-server) and 8448 (federation).
# If that's not possible for you, you can create /.well-known files to redirect
# requests. See
# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
# and
# https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
# for more information
# YOU NEED TO EDIT THIS
@ -13,10 +17,12 @@
# This is the only directory where Conduit will save its data
database_path = "/var/lib/conduit/"
database_backend = "rocksdb"
# The port Conduit will be running on. You need to set up a reverse proxy in
# your web server (e.g. apache or nginx), so all requests to /_matrix on port
# 443 and 8448 will be forwarded to the Conduit instance running on this port
# Docker users: Don't change this, you'll need to map an external port to this.
port = 6167
# Max size for uploads
@ -40,6 +46,7 @@ trusted_servers = ["matrix.org"]
#workers = 4 # default: cpu core count * 2
address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
#address = "0.0.0.0" # If Conduit is running in a container, make sure the reverse proxy (ie. Traefik) can reach it.
proxy = "none" # more examples can be found at src/database/proxy.rs:6

37
cross/README.md Normal file
View file

@ -0,0 +1,37 @@
## Cross compilation
The `cross` folder contains a set of convenience scripts (`build.sh` and `test.sh`) for cross-compiling Conduit.
Currently supported targets are
- aarch64-unknown-linux-musl
- arm-unknown-linux-musleabihf
- armv7-unknown-linux-musleabihf
- x86\_64-unknown-linux-musl
### Install prerequisites
#### Docker
[Installation guide](https://docs.docker.com/get-docker/).
```sh
$ sudo apt install docker
$ sudo systemctl start docker
$ sudo usermod -aG docker $USER
$ newgrp docker
```
#### Cross
[Installation guide](https://github.com/rust-embedded/cross/#installation).
```sh
$ cargo install cross
```
### Buiding Conduit
```sh
$ TARGET=armv7-unknown-linux-musleabihf ./cross/build.sh --release
```
The cross-compiled binary is at `target/armv7-unknown-linux-musleabihf/release/conduit`
### Testing Conduit
```sh
$ TARGET=armv7-unknown-linux-musleabihf ./cross/test.sh --release
```

99
docker/README.md
View file

@ -2,53 +2,41 @@
> **Note:** To run and use Conduit you should probably use it with a Domain or Subdomain behind a reverse proxy (like Nginx, Traefik, Apache, ...) with a Lets Encrypt certificate.
## Docker
### Build & Dockerfile
The Dockerfile provided by Conduit has two stages, each of which creates an image.
1. **Builder:** Builds the binary from local context or by cloning a git revision from the official repository.
2. **Runtime:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions.
The Dockerfile includes a few build arguments that should be supplied when building it.
``` Dockerfile
ARG LOCAL=false
ARG CREATED
ARG VERSION
ARG GIT_REF=origin/master
```
- **CREATED:** Date and time as string (date-time as defined by RFC 3339). Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.created`. Supply by it like this `$(date -u +'%Y-%m-%dT%H:%M:%SZ')`
- **VERSION:** The SemVer version of Conduit, which is in the image. Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.version`. If you have a `Cargo.toml` in your build context, you can get it with `$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)`
- **LOCAL:** *(Optional)* A boolean value, specifies if the local build context should be used, or if the official repository will be cloned. If not supplied with the build command, it will default to `false`.
- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `origin/master`.
2. **Runner:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions.
To build the image you can use the following command
``` bash
docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
```bash
docker build --tag matrixconduit/matrix-conduit:latest .
```
which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`.
**Note:** it ommits the two optional `build-arg`s.
### Run
After building the image you can simply run it with
``` bash
```bash
docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
```
or you can skip the build step and pull the image from one of the following registries:
| Registry | Image | Size |
| --------------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
| Docker Hub | [matrixconduit/matrix-conduit:latest](https://hub.docker.com/r/matrixconduit/matrix-conduit) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
| GitLab Registry | [registry.gitlab.com/famedly/conduit/conduit:latest](https://gitlab.com/famedly/conduit/container_registry/2134341) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
| Registry | Image | Size |
| --------------- | --------------------------------------------------------------- | --------------------- |
| Docker Hub | [matrixconduit/matrix-conduit:latest][dh] | ![Image Size][shield] |
| GitLab Registry | [registry.gitlab.com/famedly/conduit/matrix-conduit:latest][gl] | ![Image Size][shield] |
[dh]: https://hub.docker.com/r/matrixconduit/matrix-conduit
[gl]: https://gitlab.com/famedly/conduit/container_registry/2497937
[shield]: https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest
The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml).
You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need
@ -56,29 +44,26 @@ to pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible
If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it.
## Docker-compose
If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying
Conduit can be found [here](../DEPLOY.md).
### Build
To build the Conduit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker-compose with:
``` bash
CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) docker-compose up
```bash
docker-compose up
```
This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag. For possible `build-args`, please take a look at the above `Build & Dockerfile` section.
This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag.
### Run
If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with:
``` bash
```bash
docker-compose up -d
```
@ -101,32 +86,30 @@ So...step by step:
3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars.
4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`.
5. Create the files needed by the `well-known` service.
- `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```nginx
server {
server_name <SUBDOMAIN>.<DOMAIN>;
listen 80 default_server;
location /.well-known/matrix/ {
root /var/www;
default_type application/json;
add_header Access-Control-Allow-Origin *;
}
}
```
- `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```json
{
"m.homeserver": {
"base_url": "https://<SUBDOMAIN>.<DOMAIN>"
}
}
```
- `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```json
{
"m.server": "<SUBDOMAIN>.<DOMAIN>:443"
}
```
- `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```nginx
server {
server_name <SUBDOMAIN>.<DOMAIN>;
listen 80 default_server;
location /.well-known/matrix/server {
return 200 '{"m.server": "<SUBDOMAIN>.<DOMAIN>:443"}';
add_header Content-Type application/json;
}
location /.well-known/matrix/client {
return 200 '{"m.homeserver": {"base_url": "https://<SUBDOMAIN>.<DOMAIN>"}}';
add_header Content-Type application/json;
add_header "Access-Control-Allow-Origin" *;
}
location / {
return 404;
}
}
```
6. Run `docker-compose up -d`
7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin.

79
docker/ci-binaries-packaging.Dockerfile
View file

@ -1,51 +1,64 @@
# syntax=docker/dockerfile:1
# ---------------------------------------------------------------------------------------------------------
# This Dockerfile is intended to be built as part of Conduit's CI pipeline.
# It does not build Conduit in Docker, but just copies the matching build artifact from the build job.
# As a consequence, this is not a multiarch capable image. It always expects and packages a x86_64 binary.
# It does not build Conduit in Docker, but just copies the matching build artifact from the build jobs.
#
# It is mostly based on the normal Conduit Dockerfile, but adjusted in a few places to maximise caching.
# Credit's for the original Dockerfile: Weasy666.
# ---------------------------------------------------------------------------------------------------------
FROM alpine:3.14
FROM docker.io/alpine:3.15.0 AS runner
# Install packages needed to run Conduit
# Standard port on which Conduit launches.
# You still need to map the port when using the docker command or docker-compose.
EXPOSE 6167
# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs.
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
CONDUIT_PORT=6167
# Conduit needs:
# ca-certificates: for https
# iproute2: for `ss` for the healthcheck script
RUN apk add --no-cache \
ca-certificates \
curl \
libgcc
ca-certificates \
iproute2
ARG CREATED
ARG VERSION
ARG GIT_REF
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
# including a custom label specifying the build command
LABEL org.opencontainers.image.created=${CREATED} \
org.opencontainers.image.authors="Conduit Contributors" \
org.opencontainers.image.title="Conduit" \
org.opencontainers.image.version=${VERSION} \
org.opencontainers.image.vendor="Conduit Contributors" \
org.opencontainers.image.description="A Matrix homeserver written in Rust" \
org.opencontainers.image.url="https://conduit.rs/" \
org.opencontainers.image.revision=${GIT_REF} \
org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.documentation="" \
org.opencontainers.image.ref.name=""
org.opencontainers.image.authors="Conduit Contributors" \
org.opencontainers.image.title="Conduit" \
org.opencontainers.image.version=${VERSION} \
org.opencontainers.image.vendor="Conduit Contributors" \
org.opencontainers.image.description="A Matrix homeserver written in Rust" \
org.opencontainers.image.url="https://conduit.rs/" \
org.opencontainers.image.revision=${GIT_REF} \
org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.documentation="https://gitlab.com/famedly/conduit" \
org.opencontainers.image.ref.name=""
# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
EXPOSE 6167
# create data folder for database
# Created directory for the database and media files
RUN mkdir -p /srv/conduit/.local/share/conduit
# Copy the Conduit binary into the image at the latest possible moment to maximise caching:
COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit
COPY ./docker/healthcheck.sh /srv/conduit/
# Test if Conduit is still alive, uses the same endpoint as Element
COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
# Depending on the target platform (e.g. "linux/arm/v7", "linux/arm64/v8", or "linux/amd64")
# copy the matching binary into this docker image
ARG TARGETPLATFORM
COPY ./$TARGETPLATFORM /srv/conduit/conduit
# Improve security: Don't run stuff as root, that does not need to run as root:
# Add www-data user and group with UID 82, as used by alpine
# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
RUN set -x ; \
@ -57,13 +70,11 @@ RUN set -x ; \
RUN chown -cR www-data:www-data /srv/conduit
RUN chmod +x /srv/conduit/healthcheck.sh
# Test if Conduit is still alive, uses the same endpoint as Element
HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
# Set user to www-data
# Change user to www-data
USER www-data
# Set container home directory
WORKDIR /srv/conduit
# Run Conduit
# Run Conduit and print backtraces on panics
ENV RUST_BACKTRACE=1
ENTRYPOINT [ "/srv/conduit/conduit" ]

4
docker/docker-compose.override.traefik.yml
View file

@ -12,7 +12,7 @@ services:
- "traefik.http.routers.to-conduit.tls.certresolver=letsencrypt"
- "traefik.http.routers.to-conduit.middlewares=cors-headers@docker"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
@ -29,7 +29,7 @@ services:
- "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt"
- "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
- "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"

13
docker/healthcheck.sh
View file

@ -1,13 +1,14 @@
#!/bin/sh
# If the port is not specified as env var, take it from the config file
if [ -z ${CONDUIT_PORT} ]; then
CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')
# If the config file does not contain a default port and the CONDUIT_PORT env is not set, create
# try to get port from process list
if [ -z "${CONDUIT_PORT}" ]; then
CONDUIT_PORT=$(ss -tlpn | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*')
fi
# The actual health check.
# We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1.
# TODO: Change this to a single curl call. Do we have a config value that we can check for that?
curl --fail -s "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
curl -k --fail -s "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
# TODO: Change this to a single wget call. Do we have a config value that we can check for that?
wget --no-verbose --tries=1 --spider "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
wget --no-verbose --tries=1 --spider "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
exit 1

1
rust-toolchain
View file

@ -1 +0,0 @@
1.52

15
src/appservice_server.rs
View file

@ -1,12 +1,7 @@
use crate::{utils, Error, Result};
use bytes::BytesMut;
use ruma::api::{IncomingResponse, OutgoingRequest, SendAccessToken};
use std::{
convert::{TryFrom, TryInto},
fmt::Debug,
mem,
time::Duration,
};
use std::{fmt::Debug, mem, time::Duration};
use tracing::warn;
pub(crate) async fn send_request<T: OutgoingRequest>(
@ -21,7 +16,7 @@ where
let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();
let mut http_request = request
.try_into_http_request::<BytesMut>(&destination, SendAccessToken::IfRequired(""))
.try_into_http_request::<BytesMut>(destination, SendAccessToken::IfRequired(""))
.unwrap()
.map(|body| body.freeze());
@ -46,11 +41,7 @@ where
*reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
let url = reqwest_request.url().clone();
let mut response = globals
.reqwest_client()?
.build()?
.execute(reqwest_request)
.await?;
let mut response = globals.default_client().execute(reqwest_request).await?;
// reqwest::Response -> http::Response conversion
let status = response.status();

374
src/client_server/account.rs
View file

@ -1,34 +1,30 @@
use std::{
collections::BTreeMap,
convert::{TryFrom, TryInto},
sync::Arc,
};
use std::sync::Arc;
use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
use crate::{
database::{admin::make_user_admin, DatabaseGuard},
pdu::PduBuilder,
utils, ConduitResult, Error, Ruma,
};
use ruma::{
api::client::{
error::ErrorKind,
r0::{
account::{
change_password, deactivate, get_username_availability, register, whoami,
ThirdPartyIdRemovalStatus,
change_password, deactivate, get_3pids, get_username_availability, register,
whoami, ThirdPartyIdRemovalStatus,
},
contact::get_contacts,
uiaa::{AuthFlow, UiaaInfo},
uiaa::{AuthFlow, AuthType, UiaaInfo},
},
},
events::{
room::{
canonical_alias, guest_access, history_visibility, join_rules, member, message, name,
topic,
},
room::member::{MembershipState, RoomMemberEventContent},
EventType,
},
identifiers::RoomName,
push, RoomAliasId, RoomId, RoomVersionId, UserId,
push, UserId,
};
use tracing::info;
use serde_json::value::to_raw_value;
use tracing::{info, warn};
use register::RegistrationKind;
#[cfg(feature = "conduit_bin")]
@ -56,15 +52,16 @@ pub async fn get_register_available_route(
body: Ruma<get_username_availability::Request<'_>>,
) -> ConduitResult<get_username_availability::Response> {
// Validate user id
let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name())
.ok()
.filter(|user_id| {
!user_id.is_historical() && user_id.server_name() == db.globals.server_name()
})
.ok_or(Error::BadRequest(
ErrorKind::InvalidUsername,
"Username is invalid.",
))?;
let user_id =
UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name())
.ok()
.filter(|user_id| {
!user_id.is_historical() && user_id.server_name() == db.globals.server_name()
})
.ok_or(Error::BadRequest(
ErrorKind::InvalidUsername,
"Username is invalid.",
))?;
// Check if username is creative enough
if db.users.exists(&user_id)? {
@ -147,7 +144,7 @@ pub async fn register_route(
// UIAA
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.dummy".to_owned()],
stages: vec![AuthType::Dummy],
}],
completed: Vec::new(),
params: Default::default(),
@ -248,282 +245,16 @@ pub async fn register_route(
body.initial_device_display_name.clone(),
)?;
// If this is the first user on this server, create the admin room
if db.users.count()? == 1 {
// Create a user for the server
let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
.expect("@conduit:server_name is valid");
db.users.create(&conduit_user, None)?;
let room_id = RoomId::new(db.globals.server_name());
db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
let mutex_state = Arc::clone(
db.globals
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.or_default(),
);
let state_lock = mutex_state.lock().await;
let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone());
content.federate = true;
content.predecessor = None;
content.room_version = RoomVersionId::Version6;
// 1. The room create event
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCreate,
content: serde_json::to_value(content).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 2. Make conduit bot join
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: None,
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(conduit_user.to_string()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 3. Power levels
let mut users = BTreeMap::new();
users.insert(conduit_user.clone(), 100.into());
users.insert(user_id.clone(), 100.into());
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomPowerLevels,
content: serde_json::to_value(
ruma::events::room::power_levels::PowerLevelsEventContent {
users,
..Default::default()
},
)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.1 Join Rules
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomJoinRules,
content: serde_json::to_value(join_rules::JoinRulesEventContent::new(
join_rules::JoinRule::Invite,
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.2 History Visibility
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomHistoryVisibility,
content: serde_json::to_value(
history_visibility::HistoryVisibilityEventContent::new(
history_visibility::HistoryVisibility::Shared,
),
)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.3 Guest Access
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomGuestAccess,
content: serde_json::to_value(guest_access::GuestAccessEventContent::new(
guest_access::GuestAccess::Forbidden,
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 6. Events implied by name and topic
let room_name =
Box::<RoomName>::try_from(format!("{} Admin Room", db.globals.server_name()))
.expect("Room name is valid");
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomName,
content: serde_json::to_value(name::NameEventContent::new(Some(room_name)))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomTopic,
content: serde_json::to_value(topic::TopicEventContent {
topic: format!("Manage {}", db.globals.server_name()),
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// Room alias
let alias: RoomAliasId = format!("#admins:{}", db.globals.server_name())
.try_into()
.expect("#admins:server_name is a valid alias name");
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCanonicalAlias,
content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent {
alias: Some(alias.clone()),
alt_aliases: Vec::new(),
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
// Invite and join the real user
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Invite,
displayname: None,
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(user_id.to_string()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: Some(displayname),
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(user_id.to_string()),
redacts: None,
},
&user_id,
&room_id,
&db,
&state_lock,
)?;
// Send welcome message
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMessage,
content: serde_json::to_value(message::MessageEventContent::text_html(
"## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(),
"<h2>Thank you for trying out Conduit!</h2>\n<p>Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.</p>\n<p>Helpful links:</p>\n<blockquote>\n<p>Website: https://conduit.rs<br>Git and Documentation: https://gitlab.com/famedly/conduit<br>Report issues: https://gitlab.com/famedly/conduit/-/issues</p>\n</blockquote>\n<p>Here are some rooms you can join (by typing the command):</p>\n<p>Conduit room (Ask questions and get notified on updates):<br><code>/join #conduit:fachschaften.org</code></p>\n<p>Conduit lounge (Off-topic, only Conduit users are allowed to join)<br><code>/join #conduit-lounge:conduit.rs</code></p>\n".to_owned(),
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
}
info!("{} registered on this server", user_id);
// If this is the first real user, grant them admin privileges
// Note: the server user, @conduit:servername, is generated first
if db.users.count()? == 2 {
make_user_admin(&db, &user_id, displayname).await?;
warn!("Granting {} admin privileges as the first user", user_id);
}
db.flush()?;
Ok(register::Response {
@ -562,7 +293,7 @@ pub async fn change_password_route(
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.password".to_owned()],
stages: vec![AuthType::Password],
}],
completed: Vec::new(),
params: Default::default(),
@ -572,7 +303,7 @@ pub async fn change_password_route(
if let Some(auth) = &body.auth {
let (worked, uiaainfo) = db.uiaa.try_auth(
&sender_user,
sender_user,
sender_device,
auth,
&uiaainfo,
@ -586,24 +317,24 @@ pub async fn change_password_route(
} else if let Some(json) = body.json_body {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa
.create(&sender_user, &sender_device, &uiaainfo, &json)?;
.create(sender_user, sender_device, &uiaainfo, &json)?;
return Err(Error::Uiaa(uiaainfo));
} else {
return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
}
db.users
.set_password(&sender_user, Some(&body.new_password))?;
.set_password(sender_user, Some(&body.new_password))?;
if body.logout_devices {
// Logout all devices except the current one
for id in db
.users
.all_device_ids(&sender_user)
.all_device_ids(sender_user)
.filter_map(|id| id.ok())
.filter(|id| id != sender_device)
{
db.users.remove_device(&sender_user, &id)?;
db.users.remove_device(sender_user, &id)?;
}
}
@ -654,7 +385,7 @@ pub async fn deactivate_route(
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.password".to_owned()],
stages: vec![AuthType::Password],
}],
completed: Vec::new(),
params: Default::default(),
@ -664,8 +395,8 @@ pub async fn deactivate_route(
if let Some(auth) = &body.auth {
let (worked, uiaainfo) = db.uiaa.try_auth(
&sender_user,
&sender_device,
sender_user,
sender_device,
auth,
&uiaainfo,
&db.users,
@ -678,7 +409,7 @@ pub async fn deactivate_route(
} else if let Some(json) = body.json_body {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa
.create(&sender_user, &sender_device, &uiaainfo, &json)?;
.create(sender_user, sender_device, &uiaainfo, &json)?;
return Err(Error::Uiaa(uiaainfo));
} else {
return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
@ -688,24 +419,25 @@ pub async fn deactivate_route(
// TODO: work over federation invites
let all_rooms = db
.rooms
.rooms_joined(&sender_user)
.rooms_joined(sender_user)
.chain(
db.rooms
.rooms_invited(&sender_user)
.rooms_invited(sender_user)
.map(|t| t.map(|(r, _)| r)),
)
.collect::<Vec<_>>();
for room_id in all_rooms {
let room_id = room_id?;
let event = member::MemberEventContent {
membership: member::MembershipState::Leave,
let event = RoomMemberEventContent {
membership: MembershipState::Leave,
displayname: None,
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
join_authorized_via_users_server: None,
};
let mutex_state = Arc::clone(
@ -721,12 +453,12 @@ pub async fn deactivate_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(event).expect("event is valid, we just created it"),
content: to_raw_value(&event).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(sender_user.to_string()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -734,7 +466,7 @@ pub async fn deactivate_route(
}
// Remove devices and mark account as deactivated
db.users.deactivate_account(&sender_user)?;
db.users.deactivate_account(sender_user)?;
info!("{} deactivated their account", sender_user);
@ -756,9 +488,9 @@ pub async fn deactivate_route(
get("/_matrix/client/r0/account/3pid", data = "<body>")
)]
pub async fn third_party_route(
body: Ruma<get_contacts::Request>,
) -> ConduitResult<get_contacts::Response> {
body: Ruma<get_3pids::Request>,
) -> ConduitResult<get_3pids::Response> {
let _sender_user = body.sender_user.as_ref().expect("user is authenticated");
Ok(get_contacts::Response::new(Vec::new()).into())
Ok(get_3pids::Response::new(Vec::new()).into())
}

4
src/client_server/alias.rs
View file

@ -112,7 +112,7 @@ pub(crate) async fn get_alias_helper(
}
let mut room_id = None;
match db.rooms.id_from_alias(&room_alias)? {
match db.rooms.id_from_alias(room_alias)? {
Some(r) => room_id = Some(r),
None => {
for (_id, registration) in db.appservice.all()? {
@ -140,7 +140,7 @@ pub(crate) async fn get_alias_helper(
.await
.is_ok()
{
room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| {
room_id = Some(db.rooms.id_from_alias(room_alias)?.ok_or_else(|| {
Error::bad_config("Appservice lied to us. Room does not exist.")
})?);
break;

39
src/client_server/backup.rs
View file

@ -27,7 +27,7 @@ pub async fn create_backup_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let version = db
.key_backups
.create_backup(&sender_user, &body.algorithm, &db.globals)?;
.create_backup(sender_user, &body.algorithm, &db.globals)?;
db.flush()?;
@ -48,7 +48,7 @@ pub async fn update_backup_route(
) -> ConduitResult<update_backup::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.key_backups
.update_backup(&sender_user, &body.version, &body.algorithm, &db.globals)?;
.update_backup(sender_user, &body.version, &body.algorithm, &db.globals)?;
db.flush()?;
@ -71,7 +71,7 @@ pub async fn get_latest_backup_route(
let (version, algorithm) =
db.key_backups
.get_latest_backup(&sender_user)?
.get_latest_backup(sender_user)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Key backup does not exist.",
@ -101,7 +101,7 @@ pub async fn get_backup_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let algorithm = db
.key_backups
.get_backup(&sender_user, &body.version)?
.get_backup(sender_user, &body.version)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Key backup does not exist.",
@ -132,7 +132,7 @@ pub async fn delete_backup_route(
) -> ConduitResult<delete_backup::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.key_backups.delete_backup(&sender_user, &body.version)?;
db.key_backups.delete_backup(sender_user, &body.version)?;
db.flush()?;
@ -172,11 +172,11 @@ pub async fn add_backup_keys_route(
for (room_id, room) in &body.rooms {
for (session_id, key_data) in &room.sessions {
db.key_backups.add_key(
&sender_user,
sender_user,
&body.version,
&room_id,
&session_id,
&key_data,
room_id,
session_id,
key_data,
&db.globals,
)?
}
@ -223,11 +223,11 @@ pub async fn add_backup_key_sessions_route(
for (session_id, key_data) in &body.sessions {
db.key_backups.add_key(
&sender_user,
sender_user,
&body.version,
&body.room_id,
&session_id,
&key_data,
session_id,
key_data,
&db.globals,
)?
}
@ -272,7 +272,7 @@ pub async fn add_backup_key_session_route(
}
db.key_backups.add_key(
&sender_user,
sender_user,
&body.version,
&body.room_id,
&body.session_id,
@ -303,7 +303,7 @@ pub async fn get_backup_keys_route(
) -> ConduitResult<get_backup_keys::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let rooms = db.key_backups.get_all(&sender_user, &body.version)?;
let rooms = db.key_backups.get_all(sender_user, &body.version)?;
Ok(get_backup_keys::Response { rooms }.into())
}
@ -324,7 +324,7 @@ pub async fn get_backup_key_sessions_route(
let sessions = db
.key_backups
.get_room(&sender_user, &body.version, &body.room_id)?;
.get_room(sender_user, &body.version, &body.room_id)?;
Ok(get_backup_key_sessions::Response { sessions }.into())
}
@ -345,7 +345,7 @@ pub async fn get_backup_key_session_route(
let key_data = db
.key_backups
.get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?
.get_session(sender_user, &body.version, &body.room_id, &body.session_id)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Backup key not found for this user's session.",
@ -368,8 +368,7 @@ pub async fn delete_backup_keys_route(
) -> ConduitResult<delete_backup_keys::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.key_backups
.delete_all_keys(&sender_user, &body.version)?;
db.key_backups.delete_all_keys(sender_user, &body.version)?;
db.flush()?;
@ -395,7 +394,7 @@ pub async fn delete_backup_key_sessions_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.key_backups
.delete_room_keys(&sender_user, &body.version, &body.room_id)?;
.delete_room_keys(sender_user, &body.version, &body.room_id)?;
db.flush()?;
@ -421,7 +420,7 @@ pub async fn delete_backup_key_session_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.key_backups
.delete_room_key(&sender_user, &body.version, &body.room_id, &body.session_id)?;
.delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?;
db.flush()?;

9
src/client_server/capabilities.rs
View file

@ -1,5 +1,4 @@
use crate::ConduitResult;
use crate::Ruma;
use crate::{ConduitResult, Ruma};
use ruma::{
api::client::r0::capabilities::{
get_capabilities, Capabilities, RoomVersionStability, RoomVersionsCapability,
@ -23,12 +22,12 @@ pub async fn get_capabilities_route(
_body: Ruma<get_capabilities::Request>,
) -> ConduitResult<get_capabilities::Response> {
let mut available = BTreeMap::new();
available.insert(RoomVersionId::Version5, RoomVersionStability::Stable);
available.insert(RoomVersionId::Version6, RoomVersionStability::Stable);
available.insert(RoomVersionId::V5, RoomVersionStability::Stable);
available.insert(RoomVersionId::V6, RoomVersionStability::Stable);
let mut capabilities = Capabilities::new();
capabilities.room_versions = RoomVersionsCapability {
default: RoomVersionId::Version6,
default: RoomVersionId::V6,
available,
};

12
src/client_server/config.rs
View file

@ -30,7 +30,7 @@ pub async fn set_global_account_data_route(
) -> ConduitResult<set_global_account_data::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let data = serde_json::from_str::<serde_json::Value>(body.data.get())
let data: serde_json::Value = serde_json::from_str(body.data.get())
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
let event_type = body.event_type.to_string();
@ -68,7 +68,7 @@ pub async fn set_room_account_data_route(
) -> ConduitResult<set_room_account_data::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let data = serde_json::from_str::<serde_json::Value>(body.data.get())
let data: serde_json::Value = serde_json::from_str(body.data.get())
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
let event_type = body.event_type.to_string();
@ -103,9 +103,9 @@ pub async fn get_global_account_data_route(
) -> ConduitResult<get_global_account_data::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let event = db
let event: Box<RawJsonValue> = db
.account_data
.get::<Box<RawJsonValue>>(None, sender_user, body.event_type.clone().into())?
.get(None, sender_user, body.event_type.clone().into())?
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
let account_data = serde_json::from_str::<ExtractGlobalEventContent>(event.get())
@ -132,9 +132,9 @@ pub async fn get_room_account_data_route(
) -> ConduitResult<get_room_account_data::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let event = db
let event: Box<RawJsonValue> = db
.account_data
.get::<Box<RawJsonValue>>(
.get(
Some(&body.room_id),
sender_user,
body.event_type.clone().into(),

157
src/client_server/context.rs
View file

@ -1,6 +1,13 @@
use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
use ruma::api::client::{error::ErrorKind, r0::context::get_context};
use std::convert::TryFrom;
use ruma::{
api::client::{
error::ErrorKind,
r0::{context::get_context, filter::LazyLoadOptions},
},
events::EventType,
};
use std::{collections::HashSet, convert::TryFrom};
use tracing::error;
#[cfg(feature = "conduit_bin")]
use rocket::get;
@ -21,13 +28,19 @@ pub async fn get_context_route(
body: Ruma<get_context::Request<'_>>,
) -> ConduitResult<get_context::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
if !db.rooms.is_joined(sender_user, &body.room_id)? {
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"You don't have permission to view this room.",
));
}
// Load filter
let filter = body.filter.clone().unwrap_or_default();
let (lazy_load_enabled, lazy_load_send_redundant) = match filter.lazy_load_options {
LazyLoadOptions::Enabled {
include_redundant_members: redundant,
} => (true, redundant),
_ => (false, false),
};
let mut lazy_loaded = HashSet::new();
let base_pdu_id = db
.rooms
@ -45,12 +58,32 @@ pub async fn get_context_route(
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Base event not found.",
))?
.to_room_event();
))?;
let events_before = db
let room_id = base_event.room_id.clone();
if !db.rooms.is_joined(sender_user, &room_id)? {
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"You don't have permission to view this room.",
));
}
if !db.rooms.lazy_load_was_sent_before(
sender_user,
sender_device,
&room_id,
&base_event.sender,
)? || lazy_load_send_redundant
{
lazy_loaded.insert(base_event.sender.as_str().to_owned());
}
let base_event = base_event.to_room_event();
let events_before: Vec<_> = db
.rooms
.pdus_until(&sender_user, &body.room_id, base_token)?
.pdus_until(sender_user, &room_id, base_token)?
.take(
u32::try_from(body.limit).map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@ -58,21 +91,33 @@ pub async fn get_context_route(
/ 2,
)
.filter_map(|r| r.ok()) // Remove buggy events
.collect::<Vec<_>>();
.collect();
for (_, event) in &events_before {
if !db.rooms.lazy_load_was_sent_before(
sender_user,
sender_device,
&room_id,
&event.sender,
)? || lazy_load_send_redundant
{
lazy_loaded.insert(event.sender.as_str().to_owned());
}
}
let start_token = events_before
.last()
.and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok())
.map(|count| count.to_string());
let events_before = events_before
let events_before: Vec<_> = events_before
.into_iter()
.map(|(_, pdu)| pdu.to_room_event())
.collect::<Vec<_>>();
.collect();
let events_after = db
let events_after: Vec<_> = db
.rooms
.pdus_after(&sender_user, &body.room_id, base_token)?
.pdus_after(sender_user, &room_id, base_token)?
.take(
u32::try_from(body.limit).map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@ -80,30 +125,78 @@ pub async fn get_context_route(
/ 2,
)
.filter_map(|r| r.ok()) // Remove buggy events
.collect::<Vec<_>>();
.collect();
for (_, event) in &events_after {
if !db.rooms.lazy_load_was_sent_before(
sender_user,
sender_device,
&room_id,
&event.sender,
)? || lazy_load_send_redundant
{
lazy_loaded.insert(event.sender.as_str().to_owned());
}
}
let shortstatehash = match db.rooms.pdu_shortstatehash(
events_after
.last()
.map_or(&*body.event_id, |(_, e)| &*e.event_id),
)? {
Some(s) => s,
None => db
.rooms
.current_shortstatehash(&room_id)?
.expect("All rooms have state"),
};
let state_ids = db.rooms.state_full_ids(shortstatehash)?;
let end_token = events_after
.last()
.and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok())
.map(|count| count.to_string());
let events_after = events_after
let events_after: Vec<_> = events_after
.into_iter()
.map(|(_, pdu)| pdu.to_room_event())
.collect::<Vec<_>>();
let mut resp = get_context::Response::new();
resp.start = start_token;
resp.end = end_token;
resp.events_before = events_before;
resp.event = Some(base_event);
resp.events_after = events_after;
resp.state = db // TODO: State at event
.rooms
.room_state_full(&body.room_id)?
.values()
.map(|pdu| pdu.to_state_event())
.collect();
let mut state = Vec::new();
for (shortstatekey, id) in state_ids {
let (event_type, state_key) = db.rooms.get_statekey_from_short(shortstatekey)?;
if event_type != EventType::RoomMember {
let pdu = match db.rooms.get_pdu(&id)? {
Some(pdu) => pdu,
None => {
error!("Pdu in state not found: {}", id);
continue;
}
};
state.push(pdu.to_state_event());
} else if !lazy_load_enabled || lazy_loaded.contains(&state_key) {
let pdu = match db.rooms.get_pdu(&id)? {
Some(pdu) => pdu,
None => {
error!("Pdu in state not found: {}", id);
continue;
}
};
state.push(pdu.to_state_event());
}
}
let resp = get_context::Response {
start: start_token,
end: end_token,
events_before,
event: Some(base_event),
events_after,
state,
};
Ok(resp.into())
}

34
src/client_server/device.rs
View file

@ -3,7 +3,7 @@ use ruma::api::client::{
error::ErrorKind,
r0::{
device::{self, delete_device, delete_devices, get_device, get_devices, update_device},
uiaa::{AuthFlow, UiaaInfo},
uiaa::{AuthFlow, AuthType, UiaaInfo},
},
};
@ -25,11 +25,11 @@ pub async fn get_devices_route(
) -> ConduitResult<get_devices::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let devices = db
let devices: Vec<device::Device> = db
.users
.all_devices_metadata(sender_user)
.filter_map(|r| r.ok()) // Filter out buggy devices
.collect::<Vec<device::Device>>();
.collect();
Ok(get_devices::Response { devices }.into())
}
@ -50,7 +50,7 @@ pub async fn get_device_route(
let device = db
.users
.get_device_metadata(&sender_user, &body.body.device_id)?
.get_device_metadata(sender_user, &body.body.device_id)?
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;
Ok(get_device::Response { device }.into())
@ -72,20 +72,20 @@ pub async fn update_device_route(
let mut device = db
.users
.get_device_metadata(&sender_user, &body.device_id)?
.get_device_metadata(sender_user, &body.device_id)?
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;
device.display_name = body.display_name.clone();
db.users
.update_device_metadata(&sender_user, &body.device_id, &device)?;
.update_device_metadata(sender_user, &body.device_id, &device)?;
db.flush()?;
Ok(update_device::Response {}.into())
}
/// # `PUT /_matrix/client/r0/devices/{deviceId}`
/// # `DELETE /_matrix/client/r0/devices/{deviceId}`
///
/// Deletes the given device.
///
@ -109,7 +109,7 @@ pub async fn delete_device_route(
// UIAA
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.password".to_owned()],
stages: vec![AuthType::Password],
}],
completed: Vec::new(),
params: Default::default(),
@ -119,8 +119,8 @@ pub async fn delete_device_route(
if let Some(auth) = &body.auth {
let (worked, uiaainfo) = db.uiaa.try_auth(
&sender_user,
&sender_device,
sender_user,
sender_device,
auth,
&uiaainfo,
&db.users,
@ -133,13 +133,13 @@ pub async fn delete_device_route(
} else if let Some(json) = body.json_body {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa
.create(&sender_user, &sender_device, &uiaainfo, &json)?;
.create(sender_user, sender_device, &uiaainfo, &json)?;
return Err(Error::Uiaa(uiaainfo));
} else {
return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
}
db.users.remove_device(&sender_user, &body.device_id)?;
db.users.remove_device(sender_user, &body.device_id)?;
db.flush()?;
@ -172,7 +172,7 @@ pub async fn delete_devices_route(
// UIAA
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.password".to_owned()],
stages: vec![AuthType::Password],
}],
completed: Vec::new(),
params: Default::default(),
@ -182,8 +182,8 @@ pub async fn delete_devices_route(
if let Some(auth) = &body.auth {
let (worked, uiaainfo) = db.uiaa.try_auth(
&sender_user,
&sender_device,
sender_user,
sender_device,
auth,
&uiaainfo,
&db.users,
@ -196,14 +196,14 @@ pub async fn delete_devices_route(
} else if let Some(json) = body.json_body {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa
.create(&sender_user, &sender_device, &uiaainfo, &json)?;
.create(sender_user, sender_device, &uiaainfo, &json)?;
return Err(Error::Uiaa(uiaainfo));
} else {
return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
}
for device_id in &body.devices {
db.users.remove_device(&sender_user, &device_id)?
db.users.remove_device(sender_user, device_id)?
}
db.flush()?;

266
src/client_server/directory.rs
View file

@ -1,5 +1,3 @@
use std::convert::TryInto;
use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma};
use ruma::{
api::{
@ -17,10 +15,16 @@ use ruma::{
},
directory::{Filter, IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk, RoomNetwork},
events::{
room::{avatar, canonical_alias, guest_access, history_visibility, name, topic},
room::{
avatar::RoomAvatarEventContent,
canonical_alias::RoomCanonicalAliasEventContent,
guest_access::{GuestAccess, RoomGuestAccessEventContent},
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
name::RoomNameEventContent,
topic::RoomTopicEventContent,
},
EventType,
},
serde::Raw,
ServerName, UInt,
};
use tracing::{info, warn};
@ -161,7 +165,7 @@ pub(crate) async fn get_public_rooms_filtered_helper(
other_server,
federation::directory::get_public_rooms_filtered::v1::Request {
limit,
since: since.as_deref(),
since,
filter: Filter {
generic_search_term: filter.generic_search_term.as_deref(),
},
@ -217,167 +221,139 @@ pub(crate) async fn get_public_rooms_filtered_helper(
}
}
let mut all_rooms =
db.rooms
.public_rooms()
.map(|room_id| {
let room_id = room_id?;
let mut all_rooms: Vec<_> = db
.rooms
.public_rooms()
.map(|room_id| {
let room_id = room_id?;
let chunk = PublicRoomsChunk {
aliases: Vec::new(),
canonical_alias: db
.rooms
.room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")?
.map_or(Ok::<_, Error>(None), |s| {
Ok(serde_json::from_value::<
Raw<canonical_alias::CanonicalAliasEventContent>,
>(s.content.clone())
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
let chunk = PublicRoomsChunk {
aliases: Vec::new(),
canonical_alias: db
.rooms
.room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")?
.map_or(Ok(None), |s| {
serde_json::from_str(s.content.get())
.map(|c: RoomCanonicalAliasEventContent| c.alias)
.map_err(|_| {
Error::bad_database("Invalid canonical alias event in database.")
})?
.alias)
})?,
name: db
.rooms
.room_state_get(&room_id, &EventType::RoomName, "")?
.map_or(Ok::<_, Error>(None), |s| {
Ok(serde_json::from_value::<Raw<name::NameEventContent>>(
s.content.clone(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
})
})?,
name: db
.rooms
.room_state_get(&room_id, &EventType::RoomName, "")?
.map_or(Ok(None), |s| {
serde_json::from_str(s.content.get())
.map(|c: RoomNameEventContent| c.name)
.map_err(|_| {
Error::bad_database("Invalid room name event in database.")
})?
.name)
})?,
num_joined_members: db
.rooms
.room_joined_count(&room_id)?
.unwrap_or_else(|| {
warn!("Room {} has no member count", room_id);
0
})
.try_into()
.expect("user count should not be that big"),
topic: db
.rooms
.room_state_get(&room_id, &EventType::RoomTopic, "")?
.map_or(Ok::<_, Error>(None), |s| {
Ok(Some(
serde_json::from_value::<Raw<topic::TopicEventContent>>(
s.content.clone(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
.map_err(|_| {
Error::bad_database("Invalid room topic event in database.")
})?
.topic,
))
})?,
world_readable: db
.rooms
.room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")?
.map_or(Ok::<_, Error>(false), |s| {
Ok(serde_json::from_value::<
Raw<history_visibility::HistoryVisibilityEventContent>,
>(s.content.clone())
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
})
})?,
num_joined_members: db
.rooms
.room_joined_count(&room_id)?
.unwrap_or_else(|| {
warn!("Room {} has no member count", room_id);
0
})
.try_into()
.expect("user count should not be that big"),
topic: db
.rooms
.room_state_get(&room_id, &EventType::RoomTopic, "")?
.map_or(Ok(None), |s| {
serde_json::from_str(s.content.get())
.map(|c: RoomTopicEventContent| Some(c.topic))
.map_err(|_| {
Error::bad_database("Invalid room topic event in database.")
})
})?,
world_readable: db
.rooms
.room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")?
.map_or(Ok(false), |s| {
serde_json::from_str(s.content.get())
.map(|c: RoomHistoryVisibilityEventContent| {
c.history_visibility == HistoryVisibility::WorldReadable
})
.map_err(|_| {
Error::bad_database(
"Invalid room history visibility event in database.",
)
})?
.history_visibility
== history_visibility::HistoryVisibility::WorldReadable)
})?,
guest_can_join: db
.rooms
.room_state_get(&room_id, &EventType::RoomGuestAccess, "")?
.map_or(Ok::<_, Error>(false), |s| {
Ok(
serde_json::from_value::<Raw<guest_access::GuestAccessEventContent>>(
s.content.clone(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
})
})?,
guest_can_join: db
.rooms
.room_state_get(&room_id, &EventType::RoomGuestAccess, "")?
.map_or(Ok(false), |s| {
serde_json::from_str(s.content.get())
.map(|c: RoomGuestAccessEventContent| {
c.guest_access == GuestAccess::CanJoin
})
.map_err(|_| {
Error::bad_database("Invalid room guest access event in database.")
})?
.guest_access
== guest_access::GuestAccess::CanJoin,
)
})?,
avatar_url: db
.rooms
.room_state_get(&room_id, &EventType::RoomAvatar, "")?
.map(|s| {
Ok::<_, Error>(
serde_json::from_value::<Raw<avatar::AvatarEventContent>>(
s.content.clone(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
.map_err(|_| {
Error::bad_database("Invalid room avatar event in database.")
})?
.url,
)
})
.transpose()?
// url is now an Option<String> so we must flatten
.flatten(),
room_id,
};
Ok(chunk)
})
.filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms
.filter(|chunk| {
if let Some(query) = filter
.generic_search_term
.as_ref()
.map(|q| q.to_lowercase())
{
if let Some(name) = &chunk.name {
if name.as_str().to_lowercase().contains(&query) {
return true;
}
})
})?,
avatar_url: db
.rooms
.room_state_get(&room_id, &EventType::RoomAvatar, "")?
.map(|s| {
serde_json::from_str(s.content.get())
.map(|c: RoomAvatarEventContent| c.url)
.map_err(|_| {
Error::bad_database("Invalid room avatar event in database.")
})
})
.transpose()?
// url is now an Option<String> so we must flatten
.flatten(),
room_id,
};
Ok(chunk)
})
.filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms
.filter(|chunk| {
if let Some(query) = filter
.generic_search_term
.as_ref()
.map(|q| q.to_lowercase())
{
if let Some(name) = &chunk.name {
if name.as_str().to_lowercase().contains(&query) {
return true;
}
if let Some(topic) = &chunk.topic {
if topic.to_lowercase().contains(&query) {
return true;
}
}
if let Some(canonical_alias) = &chunk.canonical_alias {
if canonical_alias.as_str().to_lowercase().contains(&query) {
return true;
}
}
false
} else {
// No search term
true
}
})
// We need to collect all, so we can sort by member count
.collect::<Vec<_>>();
if let Some(topic) = &chunk.topic {
if topic.to_lowercase().contains(&query) {
return true;
}
}
if let Some(canonical_alias) = &chunk.canonical_alias {
if canonical_alias.as_str().to_lowercase().contains(&query) {
return true;
}
}
false
} else {
// No search term
true
}
})
// We need to collect all, so we can sort by member count
.collect();
all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members));
let total_room_count_estimate = (all_rooms.len() as u32).into();
let chunk = all_rooms
let chunk: Vec<_> = all_rooms
.into_iter()
.skip(num_since as usize)
.take(limit as usize)
.collect::<Vec<_>>();
.collect();
let prev_batch = if num_since == 0 {
None

57
src/client_server/filter.rs
View file

@ -1,32 +1,47 @@
use crate::{utils, ConduitResult};
use ruma::api::client::r0::filter::{self, create_filter, get_filter};
use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
use ruma::api::client::{
error::ErrorKind,
r0::filter::{create_filter, get_filter},
};
#[cfg(feature = "conduit_bin")]
use rocket::{get, post};
/// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}`
///
/// TODO: Loads a filter that was previously created.
#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))]
#[tracing::instrument]
pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
// TODO
Ok(get_filter::Response::new(filter::IncomingFilterDefinition {
event_fields: None,
event_format: filter::EventFormat::default(),
account_data: filter::IncomingFilter::default(),
room: filter::IncomingRoomFilter::default(),
presence: filter::IncomingFilter::default(),
})
.into())
/// Loads a filter that was previously created.
///
/// - A user can only access their own filters
#[cfg_attr(
feature = "conduit_bin",
get("/_matrix/client/r0/user/<_>/filter/<_>", data = "<body>")
)]
#[tracing::instrument(skip(db, body))]
pub async fn get_filter_route(
db: DatabaseGuard,
body: Ruma<get_filter::Request<'_>>,
) -> ConduitResult<get_filter::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let filter = match db.users.get_filter(sender_user, &body.filter_id)? {
Some(filter) => filter,
None => return Err(Error::BadRequest(ErrorKind::NotFound, "Filter not found.")),
};
Ok(get_filter::Response::new(filter).into())
}
/// # `PUT /_matrix/client/r0/user/{userId}/filter`
///
/// TODO: Creates a new filter to be used by other endpoints.
#[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))]
#[tracing::instrument]
pub async fn create_filter_route() -> ConduitResult<create_filter::Response> {
// TODO
Ok(create_filter::Response::new(utils::random_string(10)).into())
/// Creates a new filter to be used by other endpoints.
#[cfg_attr(
feature = "conduit_bin",
post("/_matrix/client/r0/user/<_>/filter", data = "<body>")
)]
#[tracing::instrument(skip(db, body))]
pub async fn create_filter_route(
db: DatabaseGuard,
body: Ruma<create_filter::Request<'_>>,
) -> ConduitResult<create_filter::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
Ok(create_filter::Response::new(db.users.create_filter(sender_user, &body.filter)?).into())
}

93
src/client_server/keys.rs
View file

@ -10,12 +10,12 @@ use ruma::{
claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures,
upload_signing_keys,
},
uiaa::{AuthFlow, UiaaInfo},
uiaa::{AuthFlow, AuthType, UiaaInfo},
},
},
federation,
},
encryption::UnsignedDeviceInfo,
serde::Raw,
DeviceId, DeviceKeyAlgorithm, UserId,
};
use serde_json::json;
@ -42,16 +42,9 @@ pub async fn upload_keys_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
if let Some(one_time_keys) = &body.one_time_keys {
for (key_key, key_value) in one_time_keys {
db.users.add_one_time_key(
sender_user,
sender_device,
key_key,
key_value,
&db.globals,
)?;
}
for (key_key, key_value) in &body.one_time_keys {
db.users
.add_one_time_key(sender_user, sender_device, key_key, key_value, &db.globals)?;
}
if let Some(device_keys) = &body.device_keys {
@ -148,7 +141,7 @@ pub async fn upload_signing_keys_route(
// UIAA
let mut uiaainfo = UiaaInfo {
flows: vec![AuthFlow {
stages: vec!["m.login.password".to_owned()],
stages: vec![AuthType::Password],
}],
completed: Vec::new(),
params: Default::default(),
@ -158,8 +151,8 @@ pub async fn upload_signing_keys_route(
if let Some(auth) = &body.auth {
let (worked, uiaainfo) = db.uiaa.try_auth(
&sender_user,
&sender_device,
sender_user,
sender_device,
auth,
&uiaainfo,
&db.users,
@ -172,7 +165,7 @@ pub async fn upload_signing_keys_route(
} else if let Some(json) = body.json_body {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa
.create(&sender_user, &sender_device, &uiaainfo, &json)?;
.create(sender_user, sender_device, &uiaainfo, &json)?;
return Err(Error::Uiaa(uiaainfo));
} else {
return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
@ -181,7 +174,7 @@ pub async fn upload_signing_keys_route(
if let Some(master_key) = &body.master_key {
db.users.add_cross_signing_keys(
sender_user,
&master_key,
master_key,
&body.self_signing_key,
&body.user_signing_key,
&db.rooms,
@ -242,10 +235,10 @@ pub async fn upload_signatures_route(
.to_owned(),
);
db.users.sign_key(
&user_id,
&key_id,
user_id,
key_id,
signature,
&sender_user,
sender_user,
&db.rooms,
&db.globals,
)?;
@ -279,7 +272,7 @@ pub async fn get_key_changes_route(
device_list_updates.extend(
db.users
.keys_changed(
&sender_user.to_string(),
sender_user.as_str(),
body.from
.parse()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?,
@ -316,7 +309,7 @@ pub async fn get_key_changes_route(
pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
sender_user: Option<&UserId>,
device_keys_input: &BTreeMap<UserId, Vec<Box<DeviceId>>>,
device_keys_input: &BTreeMap<Box<UserId>, Vec<Box<DeviceId>>>,
allowed_signatures: F,
db: &Database,
) -> Result<get_keys::Response> {
@ -328,6 +321,8 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
let mut get_over_federation = HashMap::new();
for (user_id, device_ids) in device_keys_input {
let user_id: &UserId = &**user_id;
if user_id.server_name() != db.globals.server_name() {
get_over_federation
.entry(user_id.server_name())
@ -348,59 +343,55 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
Error::bad_database("all_device_keys contained nonexistent device.")
})?;
keys.unsigned = UnsignedDeviceInfo {
device_display_name: metadata.display_name,
};
add_unsigned_device_display_name(&mut keys, metadata)
.map_err(|_| Error::bad_database("invalid device keys in database"))?;
container.insert(device_id, keys);
}
}
device_keys.insert(user_id.clone(), container);
device_keys.insert(user_id.to_owned(), container);
} else {
for device_id in device_ids {
let mut container = BTreeMap::new();
if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), &device_id)? {
let metadata = db.users.get_device_metadata(user_id, &device_id)?.ok_or(
if let Some(mut keys) = db.users.get_device_keys(user_id, device_id)? {
let metadata = db.users.get_device_metadata(user_id, device_id)?.ok_or(
Error::BadRequest(
ErrorKind::InvalidParam,
"Tried to get keys for nonexistent device.",
),
)?;
keys.unsigned = UnsignedDeviceInfo {
device_display_name: metadata.display_name,
};
container.insert(device_id.clone(), keys);
add_unsigned_device_display_name(&mut keys, metadata)
.map_err(|_| Error::bad_database("invalid device keys in database"))?;
container.insert(device_id.to_owned(), keys);
}
device_keys.insert(user_id.clone(), container);
device_keys.insert(user_id.to_owned(), container);
}
}
if let Some(master_key) = db.users.get_master_key(user_id, &allowed_signatures)? {
master_keys.insert(user_id.clone(), master_key);
master_keys.insert(user_id.to_owned(), master_key);
}
if let Some(self_signing_key) = db
.users
.get_self_signing_key(user_id, &allowed_signatures)?
{
self_signing_keys.insert(user_id.clone(), self_signing_key);
self_signing_keys.insert(user_id.to_owned(), self_signing_key);
}
if Some(user_id) == sender_user {
if let Some(user_signing_key) = db.users.get_user_signing_key(user_id)? {
user_signing_keys.insert(user_id.clone(), user_signing_key);
user_signing_keys.insert(user_id.to_owned(), user_signing_key);
}
}
}
let mut failures = BTreeMap::new();
let mut futures = get_over_federation
let mut futures: FuturesUnordered<_> = get_over_federation
.into_iter()
.map(|(server, vec)| async move {
let mut device_keys_input_fed = BTreeMap::new();
for (user_id, keys) in vec {
device_keys_input_fed.insert(user_id.clone(), keys.clone());
device_keys_input_fed.insert(user_id.to_owned(), keys.clone());
}
(
server,
@ -415,7 +406,7 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
.await,
)
})
.collect::<FuturesUnordered<_>>();
.collect();
while let Some((server, response)) = futures.next().await {
match response {
@ -439,8 +430,26 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
})
}
fn add_unsigned_device_display_name(
keys: &mut Raw<ruma::encryption::DeviceKeys>,
metadata: ruma::api::client::r0::device::Device,
) -> serde_json::Result<()> {
if let Some(display_name) = metadata.display_name {
let mut object = keys.deserialize_as::<serde_json::Map<String, serde_json::Value>>()?;
let unsigned = object.entry("unsigned").or_insert_with(|| json!({}));
if let serde_json::Value::Object(unsigned_object) = unsigned {
unsigned_object.insert("device_display_name".to_owned(), display_name.into());
}
*keys = Raw::from_json(serde_json::value::to_raw_value(&object)?);
}
Ok(())
}
pub(crate) async fn claim_keys_helper(
one_time_keys_input: &BTreeMap<UserId, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
one_time_keys_input: &BTreeMap<Box<UserId>, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
db: &Database,
) -> Result<claim_keys::Response> {
let mut one_time_keys = BTreeMap::new();

117
src/client_server/media.rs
View file

@ -1,11 +1,14 @@
use crate::{
database::media::FileMeta, database::DatabaseGuard, utils, ConduitResult, Error, Ruma,
database::{media::FileMeta, DatabaseGuard},
utils, ConduitResult, Error, Ruma,
};
use ruma::api::client::{
error::ErrorKind,
r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
r0::media::{
create_content, get_content, get_content_as_filename, get_content_thumbnail,
get_media_config,
},
};
use std::convert::TryInto;
#[cfg(feature = "conduit_bin")]
use rocket::{get, post};
@ -70,7 +73,39 @@ pub async fn create_content_route(
.into())
}
/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}`
pub async fn get_remote_content(
db: &DatabaseGuard,
mxc: &str,
server_name: &ruma::ServerName,
media_id: &str,
) -> Result<get_content::Response, Error> {
let content_response = db
.sending
.send_federation_request(
&db.globals,
server_name,
get_content::Request {
allow_remote: false,
server_name,
media_id,
},
)
.await?;
db.media
.create(
mxc.to_string(),
&db.globals,
&content_response.content_disposition.as_deref(),
&content_response.content_type.as_deref(),
&content_response.file,
)
.await?;
Ok(content_response)
}
/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}`
///
/// Load media from our server or over federation.
///
@ -99,36 +134,58 @@ pub async fn get_content_route(
}
.into())
} else if &*body.server_name != db.globals.server_name() && body.allow_remote {
let get_content_response = db
.sending
.send_federation_request(
&db.globals,
&body.server_name,
get_content::Request {
allow_remote: false,
server_name: &body.server_name,
media_id: &body.media_id,
},
)
.await?;
db.media
.create(
mxc,
&db.globals,
&get_content_response.content_disposition.as_deref(),
&get_content_response.content_type.as_deref(),
&get_content_response.file,
)
.await?;
Ok(get_content_response.into())
let remote_content_response =
get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
Ok(remote_content_response.into())
} else {
Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
}
}
/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName}`
///
/// Load media from our server or over federation, permitting desired filename.
///
/// - Only allows federation if `allow_remote` is true
#[cfg_attr(
feature = "conduit_bin",
get("/_matrix/media/r0/download/<_>/<_>/<_>", data = "<body>")
)]
#[tracing::instrument(skip(db, body))]
pub async fn get_content_as_filename_route(
db: DatabaseGuard,
body: Ruma<get_content_as_filename::Request<'_>>,
) -> ConduitResult<get_content_as_filename::Response> {
let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);
if let Some(FileMeta {
content_disposition: _,
content_type,
file,
}) = db.media.get(&db.globals, &mxc).await?
{
Ok(get_content_as_filename::Response {
file,
content_type,
content_disposition: Some(format!("inline; filename={}", body.filename)),
}
.into())
} else if &*body.server_name != db.globals.server_name() && body.allow_remote {
let remote_content_response =
get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
Ok(get_content_as_filename::Response {
content_disposition: Some(format!("inline: filename={}", body.filename)),
content_type: remote_content_response.content_type,
file: remote_content_response.file,
}
.into())
} else {
Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
}
}
/// # `GET /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
///
/// Load media thumbnail from our server or over federation.
///
@ -149,7 +206,7 @@ pub async fn get_content_thumbnail_route(
}) = db
.media
.get_thumbnail(
mxc.clone(),
&mxc,
&db.globals,
body.width
.try_into()

309
src/client_server/membership.rs
View file

@ -1,10 +1,9 @@
use crate::{
client_server,
database::DatabaseGuard,
pdu::{PduBuilder, PduEvent},
pdu::{EventHash, PduBuilder, PduEvent},
server_server, utils, ConduitResult, Database, Error, Result, Ruma,
};
use member::{MemberEventContent, MembershipState};
use ruma::{
api::{
client::{
@ -18,17 +17,20 @@ use ruma::{
federation::{self, membership::create_invite},
},
events::{
pdu::Pdu,
room::{create::CreateEventContent, member},
room::{
create::RoomCreateEventContent,
member::{MembershipState, RoomMemberEventContent},
},
EventType,
},
serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
serde::{to_canonical_value, Base64, CanonicalJsonObject, CanonicalJsonValue},
state_res::{self, RoomVersion},
uint, EventId, RoomId, RoomVersionId, ServerName, UserId,
};
use serde_json::value::{to_raw_value, RawValue as RawJsonValue};
use std::{
collections::{hash_map::Entry, BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
iter,
sync::{Arc, RwLock},
time::{Duration, Instant},
};
@ -54,25 +56,23 @@ pub async fn join_room_by_id_route(
) -> ConduitResult<join_room_by_id::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let mut servers = db
let mut servers: HashSet<_> = db
.rooms
.invite_state(&sender_user, &body.room_id)?
.invite_state(sender_user, &body.room_id)?
.unwrap_or_default()
.iter()
.filter_map(|event| {
serde_json::from_str::<serde_json::Value>(&event.json().to_string()).ok()
})
.filter_map(|event| event.get("sender").cloned())
.filter_map(|event| serde_json::from_str(event.json().get()).ok())
.filter_map(|event: serde_json::Value| event.get("sender").cloned())
.filter_map(|sender| sender.as_str().map(|s| s.to_owned()))
.filter_map(|sender| UserId::try_from(sender).ok())
.filter_map(|sender| UserId::parse(sender).ok())
.map(|user| user.server_name().to_owned())
.collect::<HashSet<_>>();
.collect();
servers.insert(body.room_id.server_name().to_owned());
let ret = join_room_by_id_helper(
&db,
body.sender_user.as_ref(),
body.sender_user.as_deref(),
&body.room_id,
&servers,
body.third_party_signed.as_ref(),
@ -99,23 +99,22 @@ pub async fn join_room_by_id_or_alias_route(
db: DatabaseGuard,
body: Ruma<join_room_by_id_or_alias::Request<'_>>,
) -> ConduitResult<join_room_by_id_or_alias::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_user = body.sender_user.as_deref().expect("user is authenticated");
let body = body.body;
let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) {
let (servers, room_id) = match Box::<RoomId>::try_from(body.room_id_or_alias) {
Ok(room_id) => {
let mut servers = db
let mut servers: HashSet<_> = db
.rooms
.invite_state(&sender_user, &room_id)?
.invite_state(sender_user, &room_id)?
.unwrap_or_default()
.iter()
.filter_map(|event| {
serde_json::from_str::<serde_json::Value>(&event.json().to_string()).ok()
})
.filter_map(|event| event.get("sender").cloned())
.filter_map(|event| serde_json::from_str(event.json().get()).ok())
.filter_map(|event: serde_json::Value| event.get("sender").cloned())
.filter_map(|sender| sender.as_str().map(|s| s.to_owned()))
.filter_map(|sender| UserId::try_from(sender).ok())
.filter_map(|sender| UserId::parse(sender).ok())
.map(|user| user.server_name().to_owned())
.collect::<HashSet<_>>();
.collect();
servers.insert(room_id.server_name().to_owned());
(servers, room_id)
@ -129,7 +128,7 @@ pub async fn join_room_by_id_or_alias_route(
let join_room_response = join_room_by_id_helper(
&db,
body.sender_user.as_ref(),
Some(sender_user),
&room_id,
&servers,
body.third_party_signed.as_ref(),
@ -204,7 +203,7 @@ pub async fn kick_user_route(
) -> ConduitResult<kick_user::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let mut event = serde_json::from_value::<Raw<ruma::events::room::member::MemberEventContent>>(
let mut event: RoomMemberEventContent = serde_json::from_str(
db.rooms
.room_state_get(
&body.room_id,
@ -216,13 +215,11 @@ pub async fn kick_user_route(
"Cannot kick member that's not in the room.",
))?
.content
.clone(),
.get(),
)
.expect("Raw::from_value always works")
.deserialize()
.map_err(|_| Error::bad_database("Invalid member event in database."))?;
event.membership = ruma::events::room::member::MembershipState::Leave;
event.membership = MembershipState::Leave;
// TODO: reason
let mutex_state = Arc::clone(
@ -238,12 +235,12 @@ pub async fn kick_user_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(event).expect("event is valid, we just created it"),
content: to_raw_value(&event).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(body.user_id.to_string()),
redacts: None,
},
&sender_user,
sender_user,
&body.room_id,
&db,
&state_lock,
@ -280,24 +277,23 @@ pub async fn ban_user_route(
&body.user_id.to_string(),
)?
.map_or(
Ok::<_, Error>(member::MemberEventContent {
membership: member::MembershipState::Ban,
Ok(RoomMemberEventContent {
membership: MembershipState::Ban,
displayname: db.users.displayname(&body.user_id)?,
avatar_url: db.users.avatar_url(&body.user_id)?,
is_direct: None,
third_party_invite: None,
blurhash: db.users.blurhash(&body.user_id)?,
reason: None,
join_authorized_via_users_server: None,
}),
|event| {
let mut event = serde_json::from_value::<Raw<member::MemberEventContent>>(
event.content.clone(),
)
.expect("Raw::from_value always works")
.deserialize()
.map_err(|_| Error::bad_database("Invalid member event in database."))?;
event.membership = ruma::events::room::member::MembershipState::Ban;
Ok(event)
serde_json::from_str(event.content.get())
.map(|event: RoomMemberEventContent| RoomMemberEventContent {
membership: MembershipState::Ban,
..event
})
.map_err(|_| Error::bad_database("Invalid member event in database."))
},
)?;
@ -314,12 +310,12 @@ pub async fn ban_user_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(event).expect("event is valid, we just created it"),
content: to_raw_value(&event).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(body.user_id.to_string()),
redacts: None,
},
&sender_user,
sender_user,
&body.room_id,
&db,
&state_lock,
@ -346,7 +342,7 @@ pub async fn unban_user_route(
) -> ConduitResult<unban_user::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let mut event = serde_json::from_value::<Raw<ruma::events::room::member::MemberEventContent>>(
let mut event: RoomMemberEventContent = serde_json::from_str(
db.rooms
.room_state_get(
&body.room_id,
@ -358,13 +354,11 @@ pub async fn unban_user_route(
"Cannot unban a user who is not banned.",
))?
.content
.clone(),
.get(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
.map_err(|_| Error::bad_database("Invalid member event in database."))?;
event.membership = ruma::events::room::member::MembershipState::Leave;
event.membership = MembershipState::Leave;
let mutex_state = Arc::clone(
db.globals
@ -379,12 +373,12 @@ pub async fn unban_user_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(event).expect("event is valid, we just created it"),
content: to_raw_value(&event).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(body.user_id.to_string()),
redacts: None,
},
&sender_user,
sender_user,
&body.room_id,
&db,
&state_lock,
@ -416,7 +410,7 @@ pub async fn forget_room_route(
) -> ConduitResult<forget_room::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.rooms.forget(&body.room_id, &sender_user)?;
db.rooms.forget(&body.room_id, sender_user)?;
db.flush()?;
@ -440,7 +434,7 @@ pub async fn joined_rooms_route(
Ok(joined_rooms::Response {
joined_rooms: db
.rooms
.rooms_joined(&sender_user)
.rooms_joined(sender_user)
.filter_map(|r| r.ok())
.collect(),
}
@ -500,7 +494,7 @@ pub async fn joined_members_route(
) -> ConduitResult<joined_members::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
if !db.rooms.is_joined(&sender_user, &body.room_id)? {
if !db.rooms.is_joined(sender_user, &body.room_id)? {
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"You aren't a member of the room.",
@ -539,13 +533,13 @@ async fn join_room_by_id_helper(
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.entry(room_id.to_owned())
.or_default(),
);
let state_lock = mutex_state.lock().await;
// Ask a remote server if we don't have this room
if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() {
if !db.rooms.exists(room_id)? && room_id.server_name() != db.globals.server_name() {
let mut make_join_response_and_server = Err(Error::BadServerResponse(
"No server available to assist in joining.",
));
@ -559,7 +553,7 @@ async fn join_room_by_id_helper(
federation::membership::create_join_event_template::v1::Request {
room_id,
user_id: sender_user,
ver: &[RoomVersionId::Version5, RoomVersionId::Version6],
ver: &[RoomVersionId::V5, RoomVersionId::V6],
},
)
.await;
@ -575,19 +569,17 @@ async fn join_room_by_id_helper(
let room_version = match make_join_response.room_version {
Some(room_version)
if room_version == RoomVersionId::Version5
|| room_version == RoomVersionId::Version6 =>
if room_version == RoomVersionId::V5 || room_version == RoomVersionId::V6 =>
{
room_version
}
_ => return Err(Error::BadServerResponse("Room version is not supported")),
};
let mut join_event_stub =
serde_json::from_str::<CanonicalJsonObject>(make_join_response.event.json().get())
.map_err(|_| {
Error::BadServerResponse("Invalid make_join event json received from server.")
})?;
let mut join_event_stub: CanonicalJsonObject =
serde_json::from_str(make_join_response.event.get()).map_err(|_| {
Error::BadServerResponse("Invalid make_join event json received from server.")
})?;
// TODO: Is origin needed?
join_event_stub.insert(
@ -604,14 +596,15 @@ async fn join_room_by_id_helper(
);
join_event_stub.insert(
"content".to_owned(),
to_canonical_value(member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: db.users.displayname(&sender_user)?,
avatar_url: db.users.avatar_url(&sender_user)?,
to_canonical_value(RoomMemberEventContent {
membership: MembershipState::Join,
displayname: db.users.displayname(sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
is_direct: None,
third_party_invite: None,
blurhash: db.users.blurhash(&sender_user)?,
blurhash: db.users.blurhash(sender_user)?,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
);
@ -629,12 +622,13 @@ async fn join_room_by_id_helper(
.expect("event is valid, we just created it");
// Generate event id
let event_id = EventId::try_from(&*format!(
let event_id = format!(
"${}",
ruma::signatures::reference_hash(&join_event_stub, &room_version)
.expect("ruma can calculate reference hashes")
))
.expect("ruma's reference hashes are valid event ids");
);
let event_id = <&EventId>::try_from(event_id.as_str())
.expect("ruma's reference hashes are valid event ids");
// Add event_id back
join_event_stub.insert(
@ -652,15 +646,15 @@ async fn join_room_by_id_helper(
remote_server,
federation::membership::create_join_event::v2::Request {
room_id,
event_id: &event_id,
pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()),
event_id,
pdu: &PduEvent::convert_to_outgoing_federation_event(join_event.clone()),
},
)
.await?;
db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
db.rooms.get_or_create_shortroomid(room_id, &db.globals)?;
let pdu = PduEvent::from_id_val(&event_id, join_event.clone())
let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone())
.map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;
let mut state = HashMap::new();
@ -670,7 +664,7 @@ async fn join_room_by_id_helper(
&send_join_response,
&room_version,
&pub_key_map,
&db,
db,
)
.await?;
@ -678,7 +672,7 @@ async fn join_room_by_id_helper(
.room_state
.state
.iter()
.map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db))
.map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db))
{
let (event_id, value) = match result {
Ok(t) => t,
@ -700,14 +694,15 @@ async fn join_room_by_id_helper(
}
let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey(
&pdu.kind,
pdu.state_key
&parsed_pdu.kind,
parsed_pdu
.state_key
.as_ref()
.expect("Pdu is a membership state event"),
&db.globals,
)?;
state.insert(incoming_shortstatekey, pdu.event_id.clone());
state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone());
let create_shortstatekey = db
.rooms
@ -723,15 +718,15 @@ async fn join_room_by_id_helper(
state
.into_iter()
.map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals))
.collect::<Result<HashSet<_>>>()?,
&db,
.collect::<Result<_>>()?,
db,
)?;
for result in send_join_response
.room_state
.auth_chain
.iter()
.map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db))
.map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db))
{
let (event_id, value) = match result {
Ok(t) => t,
@ -743,40 +738,41 @@ async fn join_room_by_id_helper(
// We append to state before appending the pdu, so we don't have a moment in time with the
// pdu without it's state. This is okay because append_pdu can't fail.
let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?;
let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?;
db.rooms.append_pdu(
&pdu,
utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"),
&[pdu.event_id.clone()],
&parsed_pdu,
join_event,
iter::once(&*parsed_pdu.event_id),
db,
)?;
// We set the room state after inserting the pdu, so that we never have a moment in time
// where events in the current room state do not exist
db.rooms.set_room_state(&room_id, statehashid)?;
db.rooms.set_room_state(room_id, statehashid)?;
} else {
let event = member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: db.users.displayname(&sender_user)?,
avatar_url: db.users.avatar_url(&sender_user)?,
let event = RoomMemberEventContent {
membership: MembershipState::Join,
displayname: db.users.displayname(sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
is_direct: None,
third_party_invite: None,
blurhash: db.users.blurhash(&sender_user)?,
blurhash: db.users.blurhash(sender_user)?,
reason: None,
join_authorized_via_users_server: None,
};
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(event).expect("event is valid, we just created it"),
content: to_raw_value(&event).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(sender_user.to_string()),
redacts: None,
},
&sender_user,
&room_id,
&db,
sender_user,
room_id,
db,
&state_lock,
)?;
}
@ -785,22 +781,22 @@ async fn join_room_by_id_helper(
db.flush()?;
Ok(join_room_by_id::Response::new(room_id.clone()).into())
Ok(join_room_by_id::Response::new(room_id.to_owned()).into())
}
fn validate_and_add_event_id(
pdu: &Raw<Pdu>,
pdu: &RawJsonValue,
room_version: &RoomVersionId,
pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, String>>>,
pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
db: &Database,
) -> Result<(EventId, CanonicalJsonObject)> {
let mut value = serde_json::from_str::<CanonicalJsonObject>(pdu.json().get()).map_err(|e| {
) -> Result<(Box<EventId>, CanonicalJsonObject)> {
let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
error!("Invalid PDU in server response: {:?}: {:?}", pdu, e);
Error::BadServerResponse("Invalid PDU in server response")
})?;
let event_id = EventId::try_from(&*format!(
let event_id = EventId::parse(format!(
"${}",
ruma::signatures::reference_hash(&value, &room_version)
ruma::signatures::reference_hash(&value, room_version)
.expect("ruma can calculate reference hashes")
))
.expect("ruma's reference hashes are valid event ids");
@ -865,32 +861,29 @@ pub(crate) async fn invite_helper<'a>(
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.entry(room_id.to_owned())
.or_default(),
);
let state_lock = mutex_state.lock().await;
let prev_events = db
let prev_events: Vec<_> = db
.rooms
.get_pdu_leaves(room_id)?
.into_iter()
.take(20)
.collect::<Vec<_>>();
.collect();
let create_event = db
.rooms
.room_state_get(room_id, &EventType::RoomCreate, "")?;
let create_event_content = create_event
let create_event_content: Option<RoomCreateEventContent> = create_event
.as_ref()
.map(|create_event| {
serde_json::from_value::<Raw<CreateEventContent>>(create_event.content.clone())
.expect("Raw::from_value always works.")
.deserialize()
.map_err(|e| {
warn!("Invalid create event: {}", e);
Error::bad_database("Invalid create event in db.")
})
serde_json::from_str(create_event.content.get()).map_err(|e| {
warn!("Invalid create event: {}", e);
Error::bad_database("Invalid create event in db.")
})
})
.transpose()?;
@ -904,13 +897,11 @@ pub(crate) async fn invite_helper<'a>(
// If there was no create event yet, assume we are creating a version 6 room right now
let room_version_id = create_event_content
.map_or(RoomVersionId::Version6, |create_event| {
create_event.room_version
});
.map_or(RoomVersionId::V6, |create_event| create_event.room_version);
let room_version =
RoomVersion::new(&room_version_id).expect("room version is supported");
let content = serde_json::to_value(MemberEventContent {
let content = to_raw_value(&RoomMemberEventContent {
avatar_url: None,
displayname: None,
is_direct: Some(is_direct),
@ -918,6 +909,7 @@ pub(crate) async fn invite_helper<'a>(
third_party_invite: None,
blurhash: None,
reason: None,
join_authorized_via_users_server: None,
})
.expect("member event is valid value");
@ -927,7 +919,7 @@ pub(crate) async fn invite_helper<'a>(
let auth_events = db.rooms.get_auth_events(
room_id,
&kind,
&sender_user,
sender_user,
Some(&state_key),
&content,
)?;
@ -946,14 +938,14 @@ pub(crate) async fn invite_helper<'a>(
unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone());
unsigned.insert(
"prev_sender".to_owned(),
serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"),
to_raw_value(&prev_pdu.sender).expect("UserId is valid"),
);
}
let pdu = PduEvent {
event_id: ruma::event_id!("$thiswillbefilledinlater"),
room_id: room_id.clone(),
sender: sender_user.clone(),
event_id: ruma::event_id!("$thiswillbefilledinlater").into(),
room_id: room_id.to_owned(),
sender: sender_user.to_owned(),
origin_server_ts: utils::millis_since_unix_epoch()
.try_into()
.expect("time is valid"),
@ -967,19 +959,23 @@ pub(crate) async fn invite_helper<'a>(
.map(|(_, pdu)| pdu.event_id.clone())
.collect(),
redacts: None,
unsigned,
hashes: ruma::events::pdu::EventHash {
unsigned: if unsigned.is_empty() {
None
} else {
Some(to_raw_value(&unsigned).expect("to_raw_value always works"))
},
hashes: EventHash {
sha256: "aaa".to_owned(),
},
signatures: BTreeMap::new(),
signatures: None,
};
let auth_check = state_res::auth_check(
&room_version,
&Arc::new(pdu.clone()),
&pdu,
create_prev_event,
None, // TODO: third_party_invite
|k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone),
None::<PduEvent>, // TODO: third_party_invite
|k, s| auth_events.get(&(k.clone(), s.to_owned())),
)
.map_err(|e| {
error!("{:?}", e);
@ -1022,12 +1018,13 @@ pub(crate) async fn invite_helper<'a>(
};
// Generate event id
let expected_event_id = EventId::try_from(&*format!(
let expected_event_id = format!(
"${}",
ruma::signatures::reference_hash(&pdu_json, &room_version_id)
.expect("ruma can calculate reference hashes")
))
.expect("ruma's reference hashes are valid event ids");
);
let expected_event_id = <&EventId>::try_from(expected_event_id.as_str())
.expect("ruma's reference hashes are valid event ids");
let response = db
.sending
@ -1035,11 +1032,11 @@ pub(crate) async fn invite_helper<'a>(
&db.globals,
user_id.server_name(),
create_invite::v2::Request {
room_id: room_id.clone(),
event_id: expected_event_id.clone(),
room_version: room_version_id,
event: PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()),
invite_room_state,
room_id,
event_id: expected_event_id,
room_version: &room_version_id,
event: &PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()),
invite_room_state: &invite_room_state,
},
)
.await?;
@ -1062,7 +1059,7 @@ pub(crate) async fn invite_helper<'a>(
warn!("Server {} changed invite event, that's not allowed in the spec: ours: {:?}, theirs: {:?}", user_id.server_name(), pdu_json, value);
}
let origin = serde_json::from_value::<Box<ServerName>>(
let origin: Box<ServerName> = serde_json::from_value(
serde_json::to_value(value.get("origin").ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Event needs an origin field.",
@ -1074,10 +1071,10 @@ pub(crate) async fn invite_helper<'a>(
let pdu_id = server_server::handle_incoming_pdu(
&origin,
&event_id,
&room_id,
room_id,
value,
true,
&db,
db,
&pub_key_map,
)
.await
@ -1092,14 +1089,13 @@ pub(crate) async fn invite_helper<'a>(
"Could not accept incoming PDU as timeline event.",
))?;
for server in db
let servers = db
.rooms
.room_servers(room_id)
.filter_map(|r| r.ok())
.filter(|server| &**server != db.globals.server_name())
{
db.sending.send_pdu(&server, &pdu_id)?;
}
.filter(|server| &**server != db.globals.server_name());
db.sending.send_pdu(servers, &pdu_id)?;
return Ok(());
}
@ -1109,7 +1105,7 @@ pub(crate) async fn invite_helper<'a>(
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.entry(room_id.to_owned())
.or_default(),
);
let state_lock = mutex_state.lock().await;
@ -1117,23 +1113,24 @@ pub(crate) async fn invite_helper<'a>(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Invite,
displayname: db.users.displayname(&user_id)?,
avatar_url: db.users.avatar_url(&user_id)?,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Invite,
displayname: db.users.displayname(user_id)?,
avatar_url: db.users.avatar_url(user_id)?,
is_direct: Some(is_direct),
third_party_invite: None,
blurhash: db.users.blurhash(&user_id)?,
blurhash: db.users.blurhash(user_id)?,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(user_id.to_string()),
redacts: None,
},
&sender_user,
sender_user,
room_id,
&db,
db,
&state_lock,
)?;

128
src/client_server/message.rs
View file

@ -5,11 +5,9 @@ use ruma::{
r0::message::{get_message_events, send_message_event},
},
events::EventType,
EventId,
};
use std::{
collections::BTreeMap,
convert::{TryFrom, TryInto},
collections::{BTreeMap, HashSet},
sync::Arc,
};
@ -45,6 +43,14 @@ pub async fn send_message_event_route(
);
let state_lock = mutex_state.lock().await;
// Forbid m.room.encrypted if encryption is disabled
if &body.event_type == "m.room.encrypted" && !db.globals.allow_encryption() {
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"Encryption has been disabled",
));
}
// Check if this is a new transaction id
if let Some(response) =
db.transaction_ids
@ -59,27 +65,26 @@ pub async fn send_message_event_route(
));
}
let event_id = EventId::try_from(
utils::string_from_bytes(&response)
.map_err(|_| Error::bad_database("Invalid txnid bytes in database."))?,
)
.map_err(|_| Error::bad_database("Invalid event id in txnid data."))?;
let event_id = utils::string_from_bytes(&response)
.map_err(|_| Error::bad_database("Invalid txnid bytes in database."))?
.try_into()
.map_err(|_| Error::bad_database("Invalid event id in txnid data."))?;
return Ok(send_message_event::Response { event_id }.into());
}
let mut unsigned = BTreeMap::new();
unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into());
unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into());
let event_id = db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::from(&body.event_type),
event_type: EventType::from(&*body.event_type),
content: serde_json::from_str(body.body.body.json().get())
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
unsigned: Some(unsigned),
state_key: None,
redacts: None,
},
&sender_user,
sender_user,
&body.room_id,
&db,
&state_lock,
@ -96,7 +101,7 @@ pub async fn send_message_event_route(
db.flush()?;
Ok(send_message_event::Response::new(event_id).into())
Ok(send_message_event::Response::new((*event_id).to_owned()).into())
}
/// # `GET /_matrix/client/r0/rooms/{roomId}/messages`
@ -115,6 +120,7 @@ pub async fn get_message_events_route(
body: Ruma<get_message_events::Request<'_>>,
) -> ConduitResult<get_message_events::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
if !db.rooms.is_joined(sender_user, &body.room_id)? {
return Err(Error::BadRequest(
@ -131,17 +137,23 @@ pub async fn get_message_events_route(
let to = body.to.as_ref().map(|t| t.parse());
db.rooms
.lazy_load_confirm_delivery(sender_user, sender_device, &body.room_id, from)?;
// Use limit or else 10
let limit = body
.limit
.try_into()
.map_or(Ok::<_, Error>(10_usize), |l: u32| Ok(l as usize))?;
let limit = body.limit.try_into().map_or(10_usize, |l: u32| l as usize);
let next_token;
let mut resp = get_message_events::Response::new();
let mut lazy_loaded = HashSet::new();
match body.dir {
get_message_events::Direction::Forward => {
let events_after = db
let events_after: Vec<_> = db
.rooms
.pdus_after(&sender_user, &body.room_id, from)?
.pdus_after(sender_user, &body.room_id, from)?
.take(limit)
.filter_map(|r| r.ok()) // Filter out buggy events
.filter_map(|(pdu_id, pdu)| {
@ -151,27 +163,34 @@ pub async fn get_message_events_route(
.ok()
})
.take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to`
.collect::<Vec<_>>();
.collect();
let end_token = events_after.last().map(|(count, _)| count.to_string());
for (_, event) in &events_after {
if !db.rooms.lazy_load_was_sent_before(
sender_user,
sender_device,
&body.room_id,
&event.sender,
)? {
lazy_loaded.insert(event.sender.clone());
}
}
let events_after = events_after
next_token = events_after.last().map(|(count, _)| count).copied();
let events_after: Vec<_> = events_after
.into_iter()
.map(|(_, pdu)| pdu.to_room_event())
.collect::<Vec<_>>();
.collect();
let mut resp = get_message_events::Response::new();
resp.start = Some(body.from.to_owned());
resp.end = end_token;
resp.start = body.from.to_owned();
resp.end = next_token.map(|count| count.to_string());
resp.chunk = events_after;
resp.state = Vec::new();
Ok(resp.into())
}
get_message_events::Direction::Backward => {
let events_before = db
let events_before: Vec<_> = db
.rooms
.pdus_until(&sender_user, &body.room_id, from)?
.pdus_until(sender_user, &body.room_id, from)?
.take(limit)
.filter_map(|r| r.ok()) // Filter out buggy events
.filter_map(|(pdu_id, pdu)| {
@ -181,22 +200,51 @@ pub async fn get_message_events_route(
.ok()
})
.take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to`
.collect::<Vec<_>>();
.collect();
let start_token = events_before.last().map(|(count, _)| count.to_string());
for (_, event) in &events_before {
if !db.rooms.lazy_load_was_sent_before(
sender_user,
sender_device,
&body.room_id,
&event.sender,
)? {
lazy_loaded.insert(event.sender.clone());
}
}
let events_before = events_before
next_token = events_before.last().map(|(count, _)| count).copied();
let events_before: Vec<_> = events_before
.into_iter()
.map(|(_, pdu)| pdu.to_room_event())
.collect::<Vec<_>>();
.collect();
let mut resp = get_message_events::Response::new();
resp.start = Some(body.from.to_owned());
resp.end = start_token;
resp.start = body.from.to_owned();
resp.end = next_token.map(|count| count.to_string());
resp.chunk = events_before;
resp.state = Vec::new();
Ok(resp.into())
}
}
resp.state = Vec::new();
for ll_id in &lazy_loaded {
if let Some(member_event) =
db.rooms
.room_state_get(&body.room_id, &EventType::RoomMember, ll_id.as_str())?
{
resp.state.push(member_event.to_state_event());
}
}
if let Some(next_token) = next_token {
db.rooms.lazy_load_mark_sent(
sender_user,
sender_device,
&body.room_id,
lazy_loaded,
next_token,
);
}
Ok(resp.into())
}

2
src/client_server/mod.rs
View file

@ -16,6 +16,7 @@ mod profile;
mod push;
mod read_marker;
mod redact;
mod report;
mod room;
mod search;
mod session;
@ -47,6 +48,7 @@ pub use profile::*;
pub use push::*;
pub use read_marker::*;
pub use redact::*;
pub use report::*;
pub use room::*;
pub use search::*;
pub use session::*;

12
src/client_server/presence.rs
View file

@ -1,6 +1,6 @@
use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma};
use ruma::api::client::r0::presence::{get_presence, set_presence};
use std::{convert::TryInto, time::Duration};
use std::time::Duration;
#[cfg(feature = "conduit_bin")]
use rocket::{get, put};
@ -19,17 +19,17 @@ pub async fn set_presence_route(
) -> ConduitResult<set_presence::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
for room_id in db.rooms.rooms_joined(&sender_user) {
for room_id in db.rooms.rooms_joined(sender_user) {
let room_id = room_id?;
db.rooms.edus.update_presence(
&sender_user,
sender_user,
&room_id,
ruma::events::presence::PresenceEvent {
content: ruma::events::presence::PresenceEventContent {
avatar_url: db.users.avatar_url(&sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
currently_active: None,
displayname: db.users.displayname(&sender_user)?,
displayname: db.users.displayname(sender_user)?,
last_active_ago: Some(
utils::millis_since_unix_epoch()
.try_into()
@ -76,7 +76,7 @@ pub async fn get_presence_route(
if let Some(presence) = db
.rooms
.edus
.get_last_presence_event(&sender_user, &room_id)?
.get_last_presence_event(sender_user, &room_id)?
{
presence_event = Some(presence);
break;

60
src/client_server/profile.rs
View file

@ -9,10 +9,10 @@ use ruma::{
},
federation::{self, query::get_profile_information::v1::ProfileField},
},
events::EventType,
serde::Raw,
events::{room::member::RoomMemberEventContent, EventType},
};
use std::{convert::TryInto, sync::Arc};
use serde_json::value::to_raw_value;
use std::sync::Arc;
#[cfg(feature = "conduit_bin")]
use rocket::{get, put};
@ -34,25 +34,25 @@ pub async fn set_displayname_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.users
.set_displayname(&sender_user, body.displayname.clone())?;
.set_displayname(sender_user, body.displayname.clone())?;
// Send a new membership event and presence update into all joined rooms
let all_rooms_joined: Vec<_> = db
.rooms
.rooms_joined(&sender_user)
.rooms_joined(sender_user)
.filter_map(|r| r.ok())
.map(|room_id| {
Ok::<_, Error>((
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
content: to_raw_value(&RoomMemberEventContent {
displayname: body.displayname.clone(),
..serde_json::from_value::<Raw<_>>(
..serde_json::from_str(
db.rooms
.room_state_get(
&room_id,
&EventType::RoomMember,
&sender_user.to_string(),
sender_user.as_str(),
)?
.ok_or_else(|| {
Error::bad_database(
@ -61,10 +61,8 @@ pub async fn set_displayname_route(
)
})?
.content
.clone(),
.get(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
.map_err(|_| Error::bad_database("Database contains invalid PDU."))?
})
.expect("event is valid, we just created it"),
@ -89,19 +87,19 @@ pub async fn set_displayname_route(
);
let state_lock = mutex_state.lock().await;
let _ =
db.rooms
.build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock);
let _ = db
.rooms
.build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock);
// Presence update
db.rooms.edus.update_presence(
&sender_user,
sender_user,
&room_id,
ruma::events::presence::PresenceEvent {
content: ruma::events::presence::PresenceEventContent {
avatar_url: db.users.avatar_url(&sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
currently_active: None,
displayname: db.users.displayname(&sender_user)?,
displayname: db.users.displayname(sender_user)?,
last_active_ago: Some(
utils::millis_since_unix_epoch()
.try_into()
@ -177,27 +175,27 @@ pub async fn set_avatar_url_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
db.users
.set_avatar_url(&sender_user, body.avatar_url.clone())?;
.set_avatar_url(sender_user, body.avatar_url.clone())?;
db.users.set_blurhash(&sender_user, body.blurhash.clone())?;
db.users.set_blurhash(sender_user, body.blurhash.clone())?;
// Send a new membership event and presence update into all joined rooms
let all_joined_rooms: Vec<_> = db
.rooms
.rooms_joined(&sender_user)
.rooms_joined(sender_user)
.filter_map(|r| r.ok())
.map(|room_id| {
Ok::<_, Error>((
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
content: to_raw_value(&RoomMemberEventContent {
avatar_url: body.avatar_url.clone(),
..serde_json::from_value::<Raw<_>>(
..serde_json::from_str(
db.rooms
.room_state_get(
&room_id,
&EventType::RoomMember,
&sender_user.to_string(),
sender_user.as_str(),
)?
.ok_or_else(|| {
Error::bad_database(
@ -206,10 +204,8 @@ pub async fn set_avatar_url_route(
)
})?
.content
.clone(),
.get(),
)
.expect("from_value::<Raw<..>> can never fail")
.deserialize()
.map_err(|_| Error::bad_database("Database contains invalid PDU."))?
})
.expect("event is valid, we just created it"),
@ -234,19 +230,19 @@ pub async fn set_avatar_url_route(
);
let state_lock = mutex_state.lock().await;
let _ =
db.rooms
.build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock);
let _ = db
.rooms
.build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock);
// Presence update
db.rooms.edus.update_presence(
&sender_user,
sender_user,
&room_id,
ruma::events::presence::PresenceEvent {
content: ruma::events::presence::PresenceEventContent {
avatar_url: db.users.avatar_url(&sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
currently_active: None,
displayname: db.users.displayname(&sender_user)?,
displayname: db.users.displayname(sender_user)?,
last_active_ago: Some(
utils::millis_since_unix_epoch()
.try_into()

80
src/client_server/push.rs
View file

@ -8,7 +8,7 @@ use ruma::{
set_pushrule_enabled, RuleKind,
},
},
events::{push_rules, EventType},
events::{push_rules::PushRulesEvent, EventType},
push::{ConditionalPushRuleInit, PatternedPushRuleInit, SimplePushRuleInit},
};
@ -29,9 +29,9 @@ pub async fn get_pushrules_all_route(
) -> ConduitResult<get_pushrules_all::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let event = db
let event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -57,9 +57,9 @@ pub async fn get_pushrule_route(
) -> ConduitResult<get_pushrule::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let event = db
let event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -105,15 +105,15 @@ pub async fn get_pushrule_route(
/// Creates a single specified push rule for this user.
#[cfg_attr(
feature = "conduit_bin",
put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<req>")
put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
)]
#[tracing::instrument(skip(db, req))]
#[tracing::instrument(skip(db, body))]
pub async fn set_pushrule_route(
db: DatabaseGuard,
req: Ruma<set_pushrule::Request<'_>>,
body: Ruma<set_pushrule::Request<'_>>,
) -> ConduitResult<set_pushrule::Response> {
let sender_user = req.sender_user.as_ref().expect("user is authenticated");
let body = req.body;
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let body = body.body;
if body.scope != "global" {
return Err(Error::BadRequest(
@ -122,9 +122,9 @@ pub async fn set_pushrule_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -193,13 +193,8 @@ pub async fn set_pushrule_route(
_ => {}
}
db.account_data.update(
None,
&sender_user,
EventType::PushRules,
&event,
&db.globals,
)?;
db.account_data
.update(None, sender_user, EventType::PushRules, &event, &db.globals)?;
db.flush()?;
@ -227,9 +222,9 @@ pub async fn get_pushrule_actions_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -289,9 +284,9 @@ pub async fn set_pushrule_actions_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -332,13 +327,8 @@ pub async fn set_pushrule_actions_route(
_ => {}
};
db.account_data.update(
None,
&sender_user,
EventType::PushRules,
&event,
&db.globals,
)?;
db.account_data
.update(None, sender_user, EventType::PushRules, &event, &db.globals)?;
db.flush()?;
@ -366,9 +356,9 @@ pub async fn get_pushrule_enabled_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -430,9 +420,9 @@ pub async fn set_pushrule_enabled_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -478,13 +468,8 @@ pub async fn set_pushrule_enabled_route(
_ => {}
}
db.account_data.update(
None,
&sender_user,
EventType::PushRules,
&event,
&db.globals,
)?;
db.account_data
.update(None, sender_user, EventType::PushRules, &event, &db.globals)?;
db.flush()?;
@ -512,9 +497,9 @@ pub async fn delete_pushrule_route(
));
}
let mut event = db
let mut event: PushRulesEvent = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
.get(None, sender_user, EventType::PushRules)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"PushRules event not found.",
@ -550,13 +535,8 @@ pub async fn delete_pushrule_route(
_ => {}
}
db.account_data.update(
None,
&sender_user,
EventType::PushRules,
&event,
&db.globals,
)?;
db.account_data
.update(None, sender_user, EventType::PushRules, &event, &db.globals)?;
db.flush()?;

14
src/client_server/read_marker.rs
View file

@ -37,7 +37,7 @@ pub async fn set_read_marker_route(
};
db.account_data.update(
Some(&body.room_id),
&sender_user,
sender_user,
EventType::FullyRead,
&fully_read_event,
&db.globals,
@ -46,7 +46,7 @@ pub async fn set_read_marker_route(
if let Some(event) = &body.read_receipt {
db.rooms.edus.private_read_set(
&body.room_id,
&sender_user,
sender_user,
db.rooms.get_pdu_count(event)?.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Event does not exist.",
@ -54,7 +54,7 @@ pub async fn set_read_marker_route(
&db.globals,
)?;
db.rooms
.reset_notification_counts(&sender_user, &body.room_id)?;
.reset_notification_counts(sender_user, &body.room_id)?;
let mut user_receipts = BTreeMap::new();
user_receipts.insert(
@ -71,7 +71,7 @@ pub async fn set_read_marker_route(
receipt_content.insert(event.to_owned(), receipts);
db.rooms.edus.readreceipt_update(
&sender_user,
sender_user,
&body.room_id,
AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
content: ruma::events::receipt::ReceiptEventContent(receipt_content),
@ -102,7 +102,7 @@ pub async fn create_receipt_route(
db.rooms.edus.private_read_set(
&body.room_id,
&sender_user,
sender_user,
db.rooms
.get_pdu_count(&body.event_id)?
.ok_or(Error::BadRequest(
@ -112,7 +112,7 @@ pub async fn create_receipt_route(
&db.globals,
)?;
db.rooms
.reset_notification_counts(&sender_user, &body.room_id)?;
.reset_notification_counts(sender_user, &body.room_id)?;
let mut user_receipts = BTreeMap::new();
user_receipts.insert(
@ -128,7 +128,7 @@ pub async fn create_receipt_route(
receipt_content.insert(body.event_id.to_owned(), receipts);
db.rooms.edus.readreceipt_update(
&sender_user,
sender_user,
&body.room_id,
AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
content: ruma::events::receipt::ReceiptEventContent(receipt_content),

11
src/client_server/redact.rs
View file

@ -3,11 +3,12 @@ use std::sync::Arc;
use crate::{database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Ruma};
use ruma::{
api::client::r0::redact::redact_event,
events::{room::redaction, EventType},
events::{room::redaction::RoomRedactionEventContent, EventType},
};
#[cfg(feature = "conduit_bin")]
use rocket::put;
use serde_json::value::to_raw_value;
/// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}`
///
@ -24,6 +25,7 @@ pub async fn redact_event_route(
body: Ruma<redact_event::Request<'_>>,
) -> ConduitResult<redact_event::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let body = body.body;
let mutex_state = Arc::clone(
db.globals
@ -38,15 +40,15 @@ pub async fn redact_event_route(
let event_id = db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomRedaction,
content: serde_json::to_value(redaction::RedactionEventContent {
content: to_raw_value(&RoomRedactionEventContent {
reason: body.reason.clone(),
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
redacts: Some(body.event_id.clone()),
redacts: Some(body.event_id.into()),
},
&sender_user,
sender_user,
&body.room_id,
&db,
&state_lock,
@ -56,5 +58,6 @@ pub async fn redact_event_route(
db.flush()?;
let event_id = (*event_id).to_owned();
Ok(redact_event::Response { event_id }.into())
}

80
src/client_server/report.rs Normal file
View file

@ -0,0 +1,80 @@
use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
use ruma::{
api::client::{error::ErrorKind, r0::room::report_content},
events::room::message,
int,
};
#[cfg(feature = "conduit_bin")]
use rocket::{http::RawStr, post};
/// # `POST /_matrix/client/r0/rooms/{roomId}/report/{eventId}`
///
/// Reports an inappropriate event to homeserver admins
///
#[cfg_attr(
feature = "conduit_bin",
post("/_matrix/client/r0/rooms/<_>/report/<_>", data = "<body>")
)]
#[tracing::instrument(skip(db, body))]
pub async fn report_event_route(
db: DatabaseGuard,
body: Ruma<report_content::Request<'_>>,
) -> ConduitResult<report_content::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let pdu = match db.rooms.get_pdu(&body.event_id)? {
Some(pdu) => pdu,
_ => {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Invalid Event ID",
))
}
};
if body.score > int!(0) || body.score < int!(-100) {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Invalid score, must be within 0 to -100",
));
};
if body.reason.chars().count() > 250 {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Reason too long, should be 250 characters or fewer",
));
};
db.admin
.send_message(message::RoomMessageEventContent::text_html(
format!(
"Report received from: {}\n\n\
Event ID: {}\n\
Room ID: {}\n\
Sent By: {}\n\n\
Report Score: {}\n\
Report Reason: {}",
sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason
),
format!(
"<details><summary>Report received from: <a href=\"https://matrix.to/#/{0}\">{0}\
</a></summary><ul><li>Event Info<ul><li>Event ID: <code>{1}</code>\
<a href=\"https://matrix.to/#/{2}/{1}\">🔗</a></li><li>Room ID: <code>{2}</code>\
</li><li>Sent By: <a href=\"https://matrix.to/#/{3}\">{3}</a></li></ul></li><li>\
Report Info<ul><li>Report Score: {4}</li><li>Report Reason: {5}</li></ul></li>\
</ul></details>",
sender_user,
pdu.event_id,
pdu.room_id,
pdu.sender,
body.score,
RawStr::new(&body.reason).html_escape()
),
));
db.flush()?;
Ok(report_content::Response {}.into())
}

314
src/client_server/room.rs
View file

@ -8,13 +8,26 @@ use ruma::{
r0::room::{self, aliases, create_room, get_room_event, upgrade_room},
},
events::{
room::{guest_access, history_visibility, join_rules, member, name, topic},
room::{
canonical_alias::RoomCanonicalAliasEventContent,
create::RoomCreateEventContent,
guest_access::{GuestAccess, RoomGuestAccessEventContent},
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
join_rules::{JoinRule, RoomJoinRulesEventContent},
member::{MembershipState, RoomMemberEventContent},
name::RoomNameEventContent,
power_levels::RoomPowerLevelsEventContent,
tombstone::RoomTombstoneEventContent,
topic::RoomTopicEventContent,
},
EventType,
},
serde::Raw,
int,
serde::{CanonicalJsonObject, JsonObject},
RoomAliasId, RoomId, RoomVersionId,
};
use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc};
use serde_json::{json, value::to_raw_value};
use std::{cmp::max, collections::BTreeMap, sync::Arc};
use tracing::{info, warn};
#[cfg(feature = "conduit_bin")]
@ -61,14 +74,26 @@ pub async fn create_room_route(
);
let state_lock = mutex_state.lock().await;
let alias: Option<RoomAliasId> =
if !db.globals.allow_room_creation()
&& !body.from_appservice
&& !db.users.is_admin(sender_user, &db.rooms, &db.globals)?
{
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"Room creation has been disabled.",
));
}
let alias: Option<Box<RoomAliasId>> =
body.room_alias_name
.as_ref()
.map_or(Ok(None), |localpart| {
// TODO: Check for invalid characters and maximum length
let alias =
RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name()))
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;
RoomAliasId::parse(format!("#{}:{}", localpart, db.globals.server_name()))
.map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias.")
})?;
if db.rooms.id_from_alias(&alias)?.is_some() {
Err(Error::BadRequest(
@ -80,12 +105,9 @@ pub async fn create_room_route(
}
})?;
let mut content = ruma::events::room::create::CreateEventContent::new(sender_user.clone());
content.federate = body.creation_content.federate;
content.predecessor = body.creation_content.predecessor.clone();
content.room_version = match body.room_version.clone() {
let room_version = match body.room_version.clone() {
Some(room_version) => {
if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 {
if room_version == RoomVersionId::V5 || room_version == RoomVersionId::V6 {
room_version
} else {
return Err(Error::BadRequest(
@ -94,19 +116,69 @@ pub async fn create_room_route(
));
}
}
None => RoomVersionId::Version6,
None => RoomVersionId::V6,
};
let content = match &body.creation_content {
Some(content) => {
let mut content = content
.deserialize_as::<CanonicalJsonObject>()
.expect("Invalid creation content");
content.insert(
"creator".into(),
json!(&sender_user).try_into().map_err(|_| {
Error::BadRequest(ErrorKind::BadJson, "Invalid creation content")
})?,
);
content.insert(
"room_version".into(),
json!(room_version.as_str()).try_into().map_err(|_| {
Error::BadRequest(ErrorKind::BadJson, "Invalid creation content")
})?,
);
content
}
None => {
let mut content = serde_json::from_str::<CanonicalJsonObject>(
to_raw_value(&RoomCreateEventContent::new(sender_user.clone()))
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid creation content"))?
.get(),
)
.unwrap();
content.insert(
"room_version".into(),
json!(room_version.as_str()).try_into().map_err(|_| {
Error::BadRequest(ErrorKind::BadJson, "Invalid creation content")
})?,
);
content
}
};
// Validate creation content
let de_result = serde_json::from_str::<CanonicalJsonObject>(
to_raw_value(&content)
.expect("Invalid creation content")
.get(),
);
if de_result.is_err() {
return Err(Error::BadRequest(
ErrorKind::BadJson,
"Invalid creation content",
));
}
// 1. The room create event
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCreate,
content: serde_json::to_value(content).expect("event is valid, we just created it"),
content: to_raw_value(&content).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -116,21 +188,22 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: db.users.displayname(&sender_user)?,
avatar_url: db.users.avatar_url(&sender_user)?,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Join,
displayname: db.users.displayname(sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
is_direct: Some(body.is_direct),
third_party_invite: None,
blurhash: db.users.blurhash(&sender_user)?,
blurhash: db.users.blurhash(sender_user)?,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(sender_user.to_string()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -149,28 +222,25 @@ pub async fn create_room_route(
});
let mut users = BTreeMap::new();
users.insert(sender_user.clone(), 100.into());
users.insert(sender_user.clone(), int!(100));
if preset == create_room::RoomPreset::TrustedPrivateChat {
for invite_ in &body.invite {
users.insert(invite_.clone(), 100.into());
users.insert(invite_.clone(), int!(100));
}
}
let mut power_levels_content =
serde_json::to_value(ruma::events::room::power_levels::PowerLevelsEventContent {
users,
..Default::default()
})
.expect("event is valid, we just created it");
let mut power_levels_content = serde_json::to_value(RoomPowerLevelsEventContent {
users,
..Default::default()
})
.expect("event is valid, we just created it");
if let Some(power_level_content_override) = &body.power_level_content_override {
let json = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
power_level_content_override.json().get(),
)
.map_err(|_| {
Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
})?;
let json: JsonObject = serde_json::from_str(power_level_content_override.json().get())
.map_err(|_| {
Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
})?;
for (key, value) in json {
power_levels_content[key] = value;
@ -180,12 +250,13 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomPowerLevels,
content: power_levels_content,
content: to_raw_value(&power_levels_content)
.expect("to_raw_value always works on serde_json::Value"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -196,18 +267,16 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCanonicalAlias,
content: serde_json::to_value(
ruma::events::room::canonical_alias::CanonicalAliasEventContent {
alias: Some(room_alias_id.clone()),
alt_aliases: vec![],
},
)
content: to_raw_value(&RoomCanonicalAliasEventContent {
alias: Some(room_alias_id.to_owned()),
alt_aliases: vec![],
})
.expect("We checked that alias earlier, it must be fine"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -220,22 +289,17 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomJoinRules,
content: match preset {
create_room::RoomPreset::PublicChat => serde_json::to_value(
join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public),
)
.expect("event is valid, we just created it"),
content: to_raw_value(&RoomJoinRulesEventContent::new(match preset {
create_room::RoomPreset::PublicChat => JoinRule::Public,
// according to spec "invite" is the default
_ => serde_json::to_value(join_rules::JoinRulesEventContent::new(
join_rules::JoinRule::Invite,
))
.expect("event is valid, we just created it"),
},
_ => JoinRule::Invite,
}))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -245,15 +309,15 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomHistoryVisibility,
content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new(
history_visibility::HistoryVisibility::Shared,
content: to_raw_value(&RoomHistoryVisibilityEventContent::new(
HistoryVisibility::Shared,
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -263,23 +327,16 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomGuestAccess,
content: match preset {
create_room::RoomPreset::PublicChat => {
serde_json::to_value(guest_access::GuestAccessEventContent::new(
guest_access::GuestAccess::Forbidden,
))
.expect("event is valid, we just created it")
}
_ => serde_json::to_value(guest_access::GuestAccessEventContent::new(
guest_access::GuestAccess::CanJoin,
))
.expect("event is valid, we just created it"),
},
content: to_raw_value(&RoomGuestAccessEventContent::new(match preset {
create_room::RoomPreset::PublicChat => GuestAccess::Forbidden,
_ => GuestAccess::CanJoin,
}))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -287,10 +344,13 @@ pub async fn create_room_route(
// 6. Events listed in initial_state
for event in &body.initial_state {
let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
let mut pdu_builder = event.deserialize_as::<PduBuilder>().map_err(|e| {
warn!("Invalid initial state event: {:?}", e);
Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.")
})?);
})?;
// Implicit state key defaults to ""
pdu_builder.state_key.get_or_insert_with(|| "".to_owned());
// Silently skip encryption events if they are not allowed
if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {
@ -298,7 +358,7 @@ pub async fn create_room_route(
}
db.rooms
.build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock)?;
.build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock)?;
}
// 7. Events implied by name and topic
@ -306,13 +366,13 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomName,
content: serde_json::to_value(name::NameEventContent::new(Some(name.clone())))
content: to_raw_value(&RoomNameEventContent::new(Some(name.clone())))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -323,7 +383,7 @@ pub async fn create_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomTopic,
content: serde_json::to_value(topic::TopicEventContent {
content: to_raw_value(&RoomTopicEventContent {
topic: topic.clone(),
})
.expect("event is valid, we just created it"),
@ -331,7 +391,7 @@ pub async fn create_room_route(
state_key: Some("".to_owned()),
redacts: None,
},
&sender_user,
sender_user,
&room_id,
&db,
&state_lock,
@ -426,7 +486,7 @@ pub async fn get_room_aliases_route(
.into())
}
/// # `GET /_matrix/client/r0/rooms/{roomId}/upgrade`
/// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade`
///
/// Upgrades the room.
///
@ -447,10 +507,7 @@ pub async fn upgrade_room_route(
) -> ConduitResult<upgrade_room::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
if !matches!(
body.new_version,
RoomVersionId::Version5 | RoomVersionId::Version6
) {
if !matches!(body.new_version, RoomVersionId::V5 | RoomVersionId::V6) {
return Err(Error::BadRequest(
ErrorKind::UnsupportedRoomVersion,
"This server does not support that room version.",
@ -477,8 +534,8 @@ pub async fn upgrade_room_route(
let tombstone_event_id = db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomTombstone,
content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent {
body: "This room has been replaced".to_string(),
content: to_raw_value(&RoomTombstoneEventContent {
body: "This room has been replaced".to_owned(),
replacement_room: replacement_room.clone(),
})
.expect("event is valid, we just created it"),
@ -504,36 +561,60 @@ pub async fn upgrade_room_route(
);
let state_lock = mutex_state.lock().await;
// Get the old room federations status
let federate = serde_json::from_value::<Raw<ruma::events::room::create::CreateEventContent>>(
// Get the old room creation event
let mut create_event_content = serde_json::from_str::<CanonicalJsonObject>(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomCreate, "")?
.ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
.content
.clone(),
.get(),
)
.expect("Raw::from_value always works")
.deserialize()
.map_err(|_| Error::bad_database("Invalid room event in database."))?
.federate;
.map_err(|_| Error::bad_database("Invalid room event in database."))?;
// Use the m.room.tombstone event as the predecessor
let predecessor = Some(ruma::events::room::create::PreviousRoom::new(
body.room_id.clone(),
tombstone_event_id,
(*tombstone_event_id).to_owned(),
));
// Send a m.room.create event containing a predecessor field and the applicable room_version
let mut create_event_content =
ruma::events::room::create::CreateEventContent::new(sender_user.clone());
create_event_content.federate = federate;
create_event_content.room_version = body.new_version.clone();
create_event_content.predecessor = predecessor;
create_event_content.insert(
"creator".into(),
json!(&sender_user)
.try_into()
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?,
);
create_event_content.insert(
"room_version".into(),
json!(&body.new_version)
.try_into()
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?,
);
create_event_content.insert(
"predecessor".into(),
json!(predecessor)
.try_into()
.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?,
);
// Validate creation event content
let de_result = serde_json::from_str::<CanonicalJsonObject>(
to_raw_value(&create_event_content)
.expect("Error forming creation event")
.get(),
);
if de_result.is_err() {
return Err(Error::BadRequest(
ErrorKind::BadJson,
"Error forming creation event",
));
}
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCreate,
content: serde_json::to_value(create_event_content)
content: to_raw_value(&create_event_content)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
@ -549,14 +630,15 @@ pub async fn upgrade_room_route(
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: serde_json::to_value(member::MemberEventContent {
membership: member::MembershipState::Join,
displayname: db.users.displayname(&sender_user)?,
avatar_url: db.users.avatar_url(&sender_user)?,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Join,
displayname: db.users.displayname(sender_user)?,
avatar_url: db.users.avatar_url(sender_user)?,
is_direct: None,
third_party_invite: None,
blurhash: db.users.blurhash(&sender_user)?,
blurhash: db.users.blurhash(sender_user)?,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
@ -611,23 +693,17 @@ pub async fn upgrade_room_route(
}
// Get the old room power levels
let mut power_levels_event_content =
serde_json::from_value::<Raw<ruma::events::room::power_levels::PowerLevelsEventContent>>(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")?
.ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
.content
.clone(),
)
.expect("database contains invalid PDU")
.deserialize()
.map_err(|_| Error::bad_database("Invalid room event in database."))?;
let mut power_levels_event_content: RoomPowerLevelsEventContent = serde_json::from_str(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")?
.ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
.content
.get(),
)
.map_err(|_| Error::bad_database("Invalid room event in database."))?;
// Setting events_default and invite to the greater of 50 and users_default + 1
let new_level = max(
50.into(),
power_levels_event_content.users_default + 1.into(),
);
let new_level = max(int!(50), power_levels_event_content.users_default + int!(1));
power_levels_event_content.events_default = new_level;
power_levels_event_content.invite = new_level;
@ -635,7 +711,7 @@ pub async fn upgrade_room_route(
let _ = db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomPowerLevels,
content: serde_json::to_value(power_levels_event_content)
content: to_raw_value(&power_levels_event_content)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),

19
src/client_server/search.rs
View file

@ -27,7 +27,7 @@ pub async fn search_events_route(
let room_ids = filter.rooms.clone().unwrap_or_else(|| {
db.rooms
.rooms_joined(&sender_user)
.rooms_joined(sender_user)
.filter_map(|r| r.ok())
.collect()
});
@ -44,11 +44,12 @@ pub async fn search_events_route(
));
}
let search = db
if let Some(search) = db
.rooms
.search_pdus(&room_id, &search_criteria.search_term)?;
searches.push(search.0.peekable());
.search_pdus(&room_id, &search_criteria.search_term)?
{
searches.push(search.0.peekable());
}
}
let skip = match body.next_batch.as_ref().map(|s| s.parse()) {
@ -74,7 +75,7 @@ pub async fn search_events_route(
}
}
let results = results
let results: Vec<_> = results
.iter()
.map(|result| {
Ok::<_, Error>(SearchResult {
@ -88,14 +89,14 @@ pub async fn search_events_route(
rank: None,
result: db
.rooms
.get_pdu_from_id(&result)?
.get_pdu_from_id(result)?
.map(|pdu| pdu.to_room_event()),
})
})
.filter_map(|r| r.ok())
.skip(skip)
.take(limit)
.collect::<Vec<_>>();
.collect();
let next_batch = if results.len() < limit as usize {
None
@ -114,7 +115,7 @@ pub async fn search_events_route(
.search_term
.split_terminator(|c: char| !c.is_alphanumeric())
.map(str::to_lowercase)
.collect::<Vec<_>>(),
.collect(),
},
})
.into())

20
src/client_server/session.rs
View file

@ -60,10 +60,10 @@ pub async fn login_route(
// Validate login method
// TODO: Other login methods
let user_id = match &body.login_info {
login::IncomingLoginInfo::Password {
login::IncomingLoginInfo::Password(login::IncomingPassword {
identifier,
password,
} => {
}) => {
let username = if let IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
matrix_id
} else {
@ -97,11 +97,11 @@ pub async fn login_route(
user_id
}
login::IncomingLoginInfo::Token { token } => {
login::IncomingLoginInfo::Token(login::IncomingToken { token }) => {
if let Some(jwt_decoding_key) = db.globals.jwt_decoding_key() {
let token = jsonwebtoken::decode::<Claims>(
&token,
&jwt_decoding_key,
token,
jwt_decoding_key,
&jsonwebtoken::Validation::default(),
)
.map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid."))?;
@ -116,6 +116,12 @@ pub async fn login_route(
));
}
}
_ => {
return Err(Error::BadRequest(
ErrorKind::Unknown,
"Unsupported login type.",
));
}
};
// Generate new device id if the user didn't specify one
@ -179,7 +185,7 @@ pub async fn logout_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
db.users.remove_device(&sender_user, sender_device)?;
db.users.remove_device(sender_user, sender_device)?;
db.flush()?;
@ -209,7 +215,7 @@ pub async fn logout_all_route(
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
for device_id in db.users.all_device_ids(sender_user).flatten() {
db.users.remove_device(&sender_user, &device_id)?;
db.users.remove_device(sender_user, &device_id)?;
}
db.flush()?;

46
src/client_server/state.rs
View file

@ -10,8 +10,8 @@ use ruma::{
},
events::{
room::{
canonical_alias::CanonicalAliasEventContent,
history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
canonical_alias::RoomCanonicalAliasEventContent,
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
},
AnyStateEventContent, EventType,
},
@ -44,7 +44,7 @@ pub async fn send_state_event_for_key_route(
&db,
sender_user,
&body.room_id,
EventType::from(&body.event_type),
EventType::from(&*body.event_type),
&body.body.body, // Yes, I hate it too
body.state_key.to_owned(),
)
@ -52,6 +52,7 @@ pub async fn send_state_event_for_key_route(
db.flush()?;
let event_id = (*event_id).to_owned();
Ok(send_state_event::Response { event_id }.into())
}
@ -73,11 +74,19 @@ pub async fn send_state_event_for_empty_key_route(
) -> ConduitResult<send_state_event::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
// Forbid m.room.encryption if encryption is disabled
if &body.event_type == "m.room.encryption" && !db.globals.allow_encryption() {
return Err(Error::BadRequest(
ErrorKind::Forbidden,
"Encryption has been disabled",
));
}
let event_id = send_state_event_for_key_helper(
&db,
sender_user,
&body.room_id,
EventType::from(&body.event_type),
EventType::from(&*body.event_type),
&body.body.body,
body.state_key.to_owned(),
)
@ -85,6 +94,7 @@ pub async fn send_state_event_for_empty_key_route(
db.flush()?;
let event_id = (*event_id).to_owned();
Ok(send_state_event::Response { event_id }.into())
}
@ -112,13 +122,13 @@ pub async fn get_state_events_route(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
.map(|event| {
serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
serde_json::from_str(event.content.get())
.map(|e: RoomHistoryVisibilityEventContent| e.history_visibility)
.map_err(|_| {
Error::bad_database(
"Invalid room history visibility event in database.",
)
})
.map(|e| e.history_visibility)
}),
Some(Ok(HistoryVisibility::WorldReadable))
)
@ -164,13 +174,13 @@ pub async fn get_state_events_for_key_route(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
.map(|event| {
serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
serde_json::from_str(event.content.get())
.map(|e: RoomHistoryVisibilityEventContent| e.history_visibility)
.map_err(|_| {
Error::bad_database(
"Invalid room history visibility event in database.",
)
})
.map(|e| e.history_visibility)
}),
Some(Ok(HistoryVisibility::WorldReadable))
)
@ -190,7 +200,7 @@ pub async fn get_state_events_for_key_route(
))?;
Ok(get_state_events_for_key::Response {
content: serde_json::from_value(event.content.clone())
content: serde_json::from_str(event.content.get())
.map_err(|_| Error::bad_database("Invalid event content in database"))?,
}
.into())
@ -220,13 +230,13 @@ pub async fn get_state_events_for_empty_key_route(
db.rooms
.room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
.map(|event| {
serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
serde_json::from_str(event.content.get())
.map(|e: RoomHistoryVisibilityEventContent| e.history_visibility)
.map_err(|_| {
Error::bad_database(
"Invalid room history visibility event in database.",
)
})
.map(|e| e.history_visibility)
}),
Some(Ok(HistoryVisibility::WorldReadable))
)
@ -246,7 +256,7 @@ pub async fn get_state_events_for_empty_key_route(
))?;
Ok(get_state_events_for_key::Response {
content: serde_json::from_value(event.content.clone())
content: serde_json::from_str(event.content.get())
.map_err(|_| Error::bad_database("Invalid event content in database"))?,
}
.into())
@ -259,13 +269,13 @@ async fn send_state_event_for_key_helper(
event_type: EventType,
json: &Raw<AnyStateEventContent>,
state_key: String,
) -> Result<EventId> {
) -> Result<Arc<EventId>> {
let sender_user = sender;
// TODO: Review this check, error if event is unparsable, use event type, allow alias if it
// previously existed
if let Ok(canonical_alias) =
serde_json::from_str::<CanonicalAliasEventContent>(json.json().get())
serde_json::from_str::<RoomCanonicalAliasEventContent>(json.json().get())
{
let mut aliases = canonical_alias.alt_aliases.clone();
@ -295,7 +305,7 @@ async fn send_state_event_for_key_helper(
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.entry(room_id.to_owned())
.or_default(),
);
let state_lock = mutex_state.lock().await;
@ -308,9 +318,9 @@ async fn send_state_event_for_key_helper(
state_key: Some(state_key),
redacts: None,
},
&sender_user,
&room_id,
&db,
sender_user,
room_id,
db,
&state_lock,
)?;

292
src/client_server/sync.rs
View file

@ -1,13 +1,19 @@
use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma, RumaResponse};
use ruma::{
api::client::r0::{sync::sync_events, uiaa::UiaaResponse},
events::{room::member::MembershipState, AnySyncEphemeralRoomEvent, EventType},
api::client::r0::{
filter::{IncomingFilterDefinition, LazyLoadOptions},
sync::sync_events,
uiaa::UiaaResponse,
},
events::{
room::member::{MembershipState, RoomMemberEventContent},
AnySyncEphemeralRoomEvent, EventType,
},
serde::Raw,
DeviceId, RoomId, UserId,
};
use std::{
collections::{hash_map::Entry, BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
sync::Arc,
time::Duration,
};
@ -33,13 +39,15 @@ use rocket::{get, tokio};
/// Calling this endpoint with a `since` parameter from a previous `next_batch` returns:
/// For joined rooms:
/// - Some of the most recent events of each timeline that happened after since
/// - If user joined the room after since: All state events and device list updates in that room
/// - If user joined the room after since: All state events (unless lazy loading is activated) and
/// all device list updates in that room
/// - If the user was already in the room: A list of all events that are in the state now, but were
/// not in the state at `since`
/// - If the state we send contains a member event: Joined and invited member counts, heroes
/// - Device list updates that happened after `since`
/// - If there are events in the timeline we send or the user send updated his read mark: Notification counts
/// - EDUs that are active now (read receipts, typing updates, presence)
/// - TODO: Allow multiple sync streams to support Pantalaimon
///
/// For invited rooms:
/// - If the user was invited after `since`: A subset of the state of the room at the point of the invite
@ -57,9 +65,10 @@ use rocket::{get, tokio};
pub async fn sync_events_route(
db: DatabaseGuard,
body: Ruma<sync_events::Request<'_>>,
) -> std::result::Result<RumaResponse<sync_events::Response>, RumaResponse<UiaaResponse>> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
) -> Result<RumaResponse<sync_events::Response>, RumaResponse<UiaaResponse>> {
let sender_user = body.sender_user.expect("user is authenticated");
let sender_device = body.sender_device.expect("user is authenticated");
let body = body.body;
let arc_db = Arc::new(db);
@ -73,34 +82,32 @@ pub async fn sync_events_route(
Entry::Vacant(v) => {
let (tx, rx) = tokio::sync::watch::channel(None);
v.insert((body.since.clone(), rx.clone()));
tokio::spawn(sync_helper_wrapper(
Arc::clone(&arc_db),
sender_user.clone(),
sender_device.clone(),
body.since.clone(),
body.full_state,
body.timeout,
body,
tx,
));
v.insert((body.since.clone(), rx)).1.clone()
rx
}
Entry::Occupied(mut o) => {
if o.get().0 != body.since {
let (tx, rx) = tokio::sync::watch::channel(None);
o.insert((body.since.clone(), rx.clone()));
tokio::spawn(sync_helper_wrapper(
Arc::clone(&arc_db),
sender_user.clone(),
sender_device.clone(),
body.since.clone(),
body.full_state,
body.timeout,
body,
tx,
));
o.insert((body.since.clone(), rx.clone()));
rx
} else {
o.get().1.clone()
@ -129,20 +136,18 @@ pub async fn sync_events_route(
async fn sync_helper_wrapper(
db: Arc<DatabaseGuard>,
sender_user: UserId,
sender_user: Box<UserId>,
sender_device: Box<DeviceId>,
since: Option<String>,
full_state: bool,
timeout: Option<Duration>,
body: sync_events::IncomingRequest,
tx: Sender<Option<ConduitResult<sync_events::Response>>>,
) {
let since = body.since.clone();
let r = sync_helper(
Arc::clone(&db),
sender_user.clone(),
sender_device.clone(),
since.clone(),
full_state,
timeout,
body,
)
.await;
@ -173,13 +178,11 @@ async fn sync_helper_wrapper(
async fn sync_helper(
db: Arc<DatabaseGuard>,
sender_user: UserId,
sender_user: Box<UserId>,
sender_device: Box<DeviceId>,
since: Option<String>,
full_state: bool,
timeout: Option<Duration>,
body: sync_events::IncomingRequest,
// bool = caching allowed
) -> std::result::Result<(sync_events::Response, bool), Error> {
) -> Result<(sync_events::Response, bool), Error> {
// TODO: match body.set_presence {
db.rooms.edus.ping_presence(&sender_user)?;
@ -189,8 +192,26 @@ async fn sync_helper(
let next_batch = db.globals.current_count()?;
let next_batch_string = next_batch.to_string();
// Load filter
let filter = match body.filter {
None => IncomingFilterDefinition::default(),
Some(sync_events::IncomingFilter::FilterDefinition(filter)) => filter,
Some(sync_events::IncomingFilter::FilterId(filter_id)) => db
.users
.get_filter(&sender_user, &filter_id)?
.unwrap_or_default(),
};
let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options {
LazyLoadOptions::Enabled {
include_redundant_members: redundant,
} => (true, redundant),
_ => (false, false),
};
let mut joined_rooms = BTreeMap::new();
let since = since
let since = body
.since
.clone()
.and_then(|string| string.parse().ok())
.unwrap_or(0);
@ -241,13 +262,13 @@ async fn sync_helper(
});
// Take the last 10 events for the timeline
let timeline_pdus = non_timeline_pdus
let timeline_pdus: Vec<_> = non_timeline_pdus
.by_ref()
.take(10)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect::<Vec<_>>();
.collect();
let send_notification_counts = !timeline_pdus.is_empty()
|| db
@ -260,6 +281,14 @@ async fn sync_helper(
// limited unless there are events in non_timeline_pdus
let limited = non_timeline_pdus.next().is_some();
let mut timeline_users = HashSet::new();
for (_, event) in &timeline_pdus {
timeline_users.insert(event.sender.as_str().to_owned());
}
db.rooms
.lazy_load_confirm_delivery(&sender_user, &sender_device, &room_id, since)?;
// Database queries:
let current_shortstatehash = db
@ -287,13 +316,13 @@ async fn sync_helper(
.filter_map(|pdu| pdu.ok()) // Ignore all broken pdus
.filter(|(_, pdu)| pdu.kind == EventType::RoomMember)
.map(|(_, pdu)| {
let content = serde_json::from_value::<
ruma::events::room::member::MemberEventContent,
>(pdu.content.clone())
.map_err(|_| Error::bad_database("Invalid member event in database."))?;
let content: RoomMemberEventContent =
serde_json::from_str(pdu.content.get()).map_err(|_| {
Error::bad_database("Invalid member event in database.")
})?;
if let Some(state_key) = &pdu.state_key {
let user_id = UserId::try_from(state_key.clone()).map_err(|_| {
let user_id = UserId::parse(state_key.clone()).map_err(|_| {
Error::bad_database("Invalid UserId in member PDU.")
})?;
@ -340,14 +369,58 @@ async fn sync_helper(
state_events,
) = if since_shortstatehash.is_none() {
// Probably since = 0, we will do an initial sync
let (joined_member_count, invited_member_count, heroes) = calculate_counts()?;
let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?;
let state_events = current_state_ids
.iter()
.map(|(_, id)| db.rooms.get_pdu(id))
.filter_map(|r| r.ok().flatten())
.collect::<Vec<_>>();
let mut state_events = Vec::new();
let mut lazy_loaded = HashSet::new();
for (shortstatekey, id) in current_state_ids {
let (event_type, state_key) = db.rooms.get_statekey_from_short(shortstatekey)?;
if event_type != EventType::RoomMember {
let pdu = match db.rooms.get_pdu(&id)? {
Some(pdu) => pdu,
None => {
error!("Pdu in state not found: {}", id);
continue;
}
};
state_events.push(pdu);
} else if !lazy_load_enabled
|| body.full_state
|| timeline_users.contains(&state_key)
{
let pdu = match db.rooms.get_pdu(&id)? {
Some(pdu) => pdu,
None => {
error!("Pdu in state not found: {}", id);
continue;
}
};
lazy_loaded.insert(
UserId::parse(state_key.as_ref())
.expect("they are in timeline_users, so they should be correct"),
);
state_events.push(pdu);
}
}
// Reset lazy loading because this is an initial sync
db.rooms
.lazy_load_reset(&sender_user, &sender_device, &room_id)?;
// The state_events above should contain all timeline_users, let's mark them as lazy
// loaded.
db.rooms.lazy_load_mark_sent(
&sender_user,
&sender_device,
&room_id,
lazy_loaded,
next_batch,
);
(
heroes,
@ -363,7 +436,7 @@ async fn sync_helper(
// Incremental /sync
let since_shortstatehash = since_shortstatehash.unwrap();
let since_sender_member = db
let since_sender_member: Option<RoomMemberEventContent> = db
.rooms
.state_get(
since_shortstatehash,
@ -371,36 +444,80 @@ async fn sync_helper(
sender_user.as_str(),
)?
.and_then(|pdu| {
serde_json::from_value::<Raw<ruma::events::room::member::MemberEventContent>>(
pdu.content.clone(),
)
.expect("Raw::from_value always works")
.deserialize()
.map_err(|_| Error::bad_database("Invalid PDU in database."))
.ok()
serde_json::from_str(pdu.content.get())
.map_err(|_| Error::bad_database("Invalid PDU in database."))
.ok()
});
let joined_since_last_sync = since_sender_member
.map_or(true, |member| member.membership != MembershipState::Join);
let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?;
let mut state_events = Vec::new();
let mut lazy_loaded = HashSet::new();
let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?;
if since_shortstatehash != current_shortstatehash {
let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?;
let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?;
let state_events = if joined_since_last_sync {
current_state_ids
.iter()
.map(|(_, id)| db.rooms.get_pdu(id))
.filter_map(|r| r.ok().flatten())
.collect::<Vec<_>>()
} else {
current_state_ids
.iter()
.filter(|(key, id)| since_state_ids.get(key) != Some(id))
.map(|(_, id)| db.rooms.get_pdu(id))
.filter_map(|r| r.ok().flatten())
.collect()
};
for (key, id) in current_state_ids {
if body.full_state || since_state_ids.get(&key) != Some(&id) {
let pdu = match db.rooms.get_pdu(&id)? {
Some(pdu) => pdu,
None => {
error!("Pdu in state not found: {}", id);
continue;
}
};
if pdu.kind == EventType::RoomMember {
match UserId::parse(
pdu.state_key
.as_ref()
.expect("State event has state key")
.clone(),
) {
Ok(state_key_userid) => {
lazy_loaded.insert(state_key_userid);
}
Err(e) => error!("Invalid state key for member event: {}", e),
}
}
state_events.push(pdu);
}
}
}
for (_, event) in &timeline_pdus {
if lazy_loaded.contains(&event.sender) {
continue;
}
if !db.rooms.lazy_load_was_sent_before(
&sender_user,
&sender_device,
&room_id,
&event.sender,
)? || lazy_load_send_redundant
{
if let Some(member_event) = db.rooms.room_state_get(
&room_id,
&EventType::RoomMember,
event.sender.as_str(),
)? {
lazy_loaded.insert(event.sender.clone());
state_events.push(member_event);
}
}
}
db.rooms.lazy_load_mark_sent(
&sender_user,
&sender_device,
&room_id,
lazy_loaded,
next_batch,
);
let encrypted_room = db
.rooms
@ -425,18 +542,16 @@ async fn sync_helper(
}
if let Some(state_key) = &state_event.state_key {
let user_id = UserId::try_from(state_key.clone())
let user_id = UserId::parse(state_key.clone())
.map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?;
if user_id == sender_user {
continue;
}
let new_membership = serde_json::from_value::<
Raw<ruma::events::room::member::MemberEventContent>,
>(state_event.content.clone())
.expect("Raw::from_value always works")
.deserialize()
let new_membership = serde_json::from_str::<RoomMemberEventContent>(
state_event.content.get(),
)
.map_err(|_| Error::bad_database("Invalid PDU in database."))?
.membership;
@ -525,18 +640,18 @@ async fn sync_helper(
Ok(Some(db.rooms.pdu_count(pdu_id)?.to_string()))
})?;
let room_events = timeline_pdus
let room_events: Vec<_> = timeline_pdus
.iter()
.map(|(_, pdu)| pdu.to_sync_room_event())
.collect::<Vec<_>>();
.collect();
let mut edus = db
let mut edus: Vec<_> = db
.rooms
.edus
.readreceipts_since(&room_id, since)
.filter_map(|r| r.ok()) // Filter out buggy events
.map(|(_, _, v)| v)
.collect::<Vec<_>>();
.collect();
if db.rooms.edus.last_typing_update(&room_id, &db.globals)? > since {
edus.push(
@ -565,7 +680,7 @@ async fn sync_helper(
.map_err(|_| Error::bad_database("Invalid account event in database."))
.ok()
})
.collect::<Vec<_>>(),
.collect(),
},
summary: sync_events::RoomSummary {
heroes,
@ -630,7 +745,7 @@ async fn sync_helper(
}
let mut left_rooms = BTreeMap::new();
let all_left_rooms = db.rooms.rooms_left(&sender_user).collect::<Vec<_>>();
let all_left_rooms: Vec<_> = db.rooms.rooms_left(&sender_user).collect();
for result in all_left_rooms {
let (room_id, left_state_events) = result?;
@ -670,7 +785,7 @@ async fn sync_helper(
}
let mut invited_rooms = BTreeMap::new();
let all_invited_rooms = db.rooms.rooms_invited(&sender_user).collect::<Vec<_>>();
let all_invited_rooms: Vec<_> = db.rooms.rooms_invited(&sender_user).collect();
for result in all_invited_rooms {
let (room_id, invite_state_events) = result?;
@ -739,7 +854,7 @@ async fn sync_helper(
presence: sync_events::Presence {
events: presence_updates
.into_iter()
.map(|(_, v)| Raw::from(v))
.map(|(_, v)| Raw::new(&v).expect("PresenceEvent always serializes successfully"))
.collect(),
},
account_data: sync_events::GlobalAccountData {
@ -752,38 +867,33 @@ async fn sync_helper(
.map_err(|_| Error::bad_database("Invalid account event in database."))
.ok()
})
.collect::<Vec<_>>(),
.collect(),
},
device_lists: sync_events::DeviceLists {
changed: device_list_updates.into_iter().collect(),
left: device_list_left.into_iter().collect(),
},
device_one_time_keys_count: if db.users.last_one_time_keys_update(&sender_user)? > since
|| since == 0
{
db.users.count_one_time_keys(&sender_user, &sender_device)?
} else {
BTreeMap::new()
},
device_one_time_keys_count: db.users.count_one_time_keys(&sender_user, &sender_device)?,
to_device: sync_events::ToDevice {
events: db
.users
.get_to_device_events(&sender_user, &sender_device)?,
},
// Fallback keys are not yet supported
device_unused_fallback_key_types: None,
};
// TODO: Retry the endpoint instead of returning (waiting for #118)
if !full_state
if !body.full_state
&& response.rooms.is_empty()
&& response.presence.is_empty()
&& response.account_data.is_empty()
&& response.device_lists.is_empty()
&& response.device_one_time_keys_count.is_empty()
&& response.to_device.is_empty()
{
// Hang a few seconds so requests are not spammed
// Stop hanging if new info arrives
let mut duration = timeout.unwrap_or_default();
let mut duration = body.timeout.unwrap_or_default();
if duration.as_secs() > 30 {
duration = Duration::from_secs(30);
}
@ -803,7 +913,7 @@ fn share_encrypted_room(
) -> Result<bool> {
Ok(db
.rooms
.get_shared_rooms(vec![sender_user.clone(), user_id.clone()])?
.get_shared_rooms(vec![sender_user.to_owned(), user_id.to_owned()])?
.filter_map(|r| r.ok())
.filter(|room_id| room_id != ignore_room)
.filter_map(|other_room_id| {

23
src/client_server/tag.rs
View file

@ -1,7 +1,10 @@
use crate::{database::DatabaseGuard, ConduitResult, Ruma};
use ruma::{
api::client::r0::tag::{create_tag, delete_tag, get_tags},
events::EventType,
events::{
tag::{TagEvent, TagEventContent},
EventType,
},
};
use std::collections::BTreeMap;
@ -26,9 +29,9 @@ pub async fn update_tag_route(
let mut tags_event = db
.account_data
.get::<ruma::events::tag::TagEvent>(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| ruma::events::tag::TagEvent {
content: ruma::events::tag::TagEventContent {
.get(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| TagEvent {
content: TagEventContent {
tags: BTreeMap::new(),
},
});
@ -68,9 +71,9 @@ pub async fn delete_tag_route(
let mut tags_event = db
.account_data
.get::<ruma::events::tag::TagEvent>(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| ruma::events::tag::TagEvent {
content: ruma::events::tag::TagEventContent {
.get(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| TagEvent {
content: TagEventContent {
tags: BTreeMap::new(),
},
});
@ -108,9 +111,9 @@ pub async fn get_tags_route(
Ok(get_tags::Response {
tags: db
.account_data
.get::<ruma::events::tag::TagEvent>(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| ruma::events::tag::TagEvent {
content: ruma::events::tag::TagEventContent {
.get(Some(&body.room_id), sender_user, EventType::Tag)?
.unwrap_or_else(|| TagEvent {
content: TagEventContent {
tags: BTreeMap::new(),
},
})

10
src/client_server/to_device.rs
View file

@ -53,7 +53,7 @@ pub async fn send_event_to_device_route(
serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(
DirectDeviceContent {
sender: sender_user.clone(),
ev_type: EventType::from(&body.event_type),
ev_type: EventType::from(&*body.event_type),
message_id: body.txn_id.clone(),
messages,
},
@ -68,8 +68,8 @@ pub async fn send_event_to_device_route(
match target_device_id_maybe {
DeviceIdOrAllDevices::DeviceId(target_device_id) => db.users.add_to_device_event(
sender_user,
&target_user_id,
&target_device_id,
target_user_id,
target_device_id,
&body.event_type,
event.deserialize_as().map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
@ -78,10 +78,10 @@ pub async fn send_event_to_device_route(
)?,
DeviceIdOrAllDevices::AllDevices => {
for target_device_id in db.users.all_device_ids(&target_user_id) {
for target_device_id in db.users.all_device_ids(target_user_id) {
db.users.add_to_device_event(
sender_user,
&target_user_id,
target_user_id,
&target_device_id?,
&body.event_type,
event.deserialize_as().map_err(|_| {

4
src/client_server/typing.rs
View file

@ -21,7 +21,7 @@ pub fn create_typing_event_route(
if let Typing::Yes(duration) = body.state {
db.rooms.edus.typing_add(
&sender_user,
sender_user,
&body.room_id,
duration.as_millis() as u64 + utils::millis_since_unix_epoch(),
&db.globals,
@ -29,7 +29,7 @@ pub fn create_typing_event_route(
} else {
db.rooms
.edus
.typing_remove(&sender_user, &body.room_id, &db.globals)?;
.typing_remove(sender_user, &body.room_id, &db.globals)?;
}
Ok(create_typing_event::Response {}.into())

11
src/client_server/unversioned.rs
View file

@ -1,3 +1,5 @@
use std::{collections::BTreeMap, iter::FromIterator};
use crate::ConduitResult;
use ruma::api::client::unversioned::get_supported_versions;
@ -17,11 +19,10 @@ use rocket::get;
#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))]
#[tracing::instrument]
pub async fn get_supported_versions_route() -> ConduitResult<get_supported_versions::Response> {
let mut resp =
get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]);
resp.unstable_features
.insert("org.matrix.e2e_cross_signing".to_owned(), true);
let resp = get_supported_versions::Response {
versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()],
unstable_features: BTreeMap::from_iter([("org.matrix.e2e_cross_signing".to_owned(), true)]),
};
Ok(resp.into())
}

55
src/client_server/voip.rs
View file

@ -1,6 +1,11 @@
use crate::ConduitResult;
use crate::{database::DatabaseGuard, ConduitResult, Ruma};
use hmac::{Hmac, Mac, NewMac};
use ruma::api::client::r0::voip::get_turn_server_info;
use std::time::Duration;
use ruma::SecondsSinceUnixEpoch;
use sha1::Sha1;
use std::time::{Duration, SystemTime};
type HmacSha1 = Hmac<Sha1>;
#[cfg(feature = "conduit_bin")]
use rocket::get;
@ -8,14 +13,46 @@ use rocket::get;
/// # `GET /_matrix/client/r0/voip/turnServer`
///
/// TODO: Returns information about the recommended turn server.
#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))]
#[tracing::instrument]
pub async fn turn_server_route() -> ConduitResult<get_turn_server_info::Response> {
#[cfg_attr(
feature = "conduit_bin",
get("/_matrix/client/r0/voip/turnServer", data = "<body>")
)]
#[tracing::instrument(skip(body, db))]
pub async fn turn_server_route(
body: Ruma<get_turn_server_info::Request>,
db: DatabaseGuard,
) -> ConduitResult<get_turn_server_info::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let turn_secret = db.globals.turn_secret();
let (username, password) = if !turn_secret.is_empty() {
let expiry = SecondsSinceUnixEpoch::from_system_time(
SystemTime::now() + Duration::from_secs(db.globals.turn_ttl()),
)
.expect("time is valid");
let username: String = format!("{}:{}", expiry.get(), sender_user);
let mut mac = HmacSha1::new_from_slice(turn_secret.as_bytes())
.expect("HMAC can take key of any size");
mac.update(username.as_bytes());
let password: String = base64::encode_config(mac.finalize().into_bytes(), base64::STANDARD);
(username, password)
} else {
(
db.globals.turn_username().clone(),
db.globals.turn_password().clone(),
)
};
Ok(get_turn_server_info::Response {
username: "".to_owned(),
password: "".to_owned(),
uris: Vec::new(),
ttl: Duration::from_secs(60 * 60 * 24),
username,
password,
uris: db.globals.turn_uris().to_vec(),
ttl: Duration::from_secs(db.globals.turn_ttl()),
}
.into())
}

131
src/config.rs Normal file
View file

@ -0,0 +1,131 @@
use std::collections::BTreeMap;
use ruma::ServerName;
use serde::{de::IgnoredAny, Deserialize};
use tracing::warn;
mod proxy;
use self::proxy::ProxyConfig;
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
pub server_name: Box<ServerName>,
#[serde(default = "default_database_backend")]
pub database_backend: String,
pub database_path: String,
#[serde(default = "default_db_cache_capacity_mb")]
pub db_cache_capacity_mb: f64,
#[serde(default = "default_conduit_cache_capacity_modifier")]
pub conduit_cache_capacity_modifier: f64,
#[serde(default = "default_rocksdb_max_open_files")]
pub rocksdb_max_open_files: i32,
#[serde(default = "default_pdu_cache_capacity")]
pub pdu_cache_capacity: u32,
#[serde(default = "default_cleanup_second_interval")]
pub cleanup_second_interval: u32,
#[serde(default = "default_max_request_size")]
pub max_request_size: u32,
#[serde(default = "default_max_concurrent_requests")]
pub max_concurrent_requests: u16,
#[serde(default = "false_fn")]
pub allow_registration: bool,
#[serde(default = "true_fn")]
pub allow_encryption: bool,
#[serde(default = "false_fn")]
pub allow_federation: bool,
#[serde(default = "true_fn")]
pub allow_room_creation: bool,
#[serde(default = "false_fn")]
pub allow_jaeger: bool,
#[serde(default = "false_fn")]
pub tracing_flame: bool,
#[serde(default)]
pub proxy: ProxyConfig,
pub jwt_secret: Option<String>,
#[serde(default = "Vec::new")]
pub trusted_servers: Vec<Box<ServerName>>,
#[serde(default = "default_log")]
pub log: String,
#[serde(default)]
pub turn_username: String,
#[serde(default)]
pub turn_password: String,
#[serde(default = "Vec::new")]
pub turn_uris: Vec<String>,
#[serde(default)]
pub turn_secret: String,
#[serde(default = "default_turn_ttl")]
pub turn_ttl: u64,
#[serde(flatten)]
pub catchall: BTreeMap<String, IgnoredAny>,
}
const DEPRECATED_KEYS: &[&str] = &["cache_capacity"];
impl Config {
pub fn warn_deprecated(&self) {
let mut was_deprecated = false;
for key in self
.catchall
.keys()
.filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
{
warn!("Config parameter {} is deprecated", key);
was_deprecated = true;
}
if was_deprecated {
warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
}
}
}
fn false_fn() -> bool {
false
}
fn true_fn() -> bool {
true
}
fn default_database_backend() -> String {
"sqlite".to_owned()
}
fn default_db_cache_capacity_mb() -> f64 {
10.0
}
fn default_conduit_cache_capacity_modifier() -> f64 {
1.0
}
fn default_rocksdb_max_open_files() -> i32 {
20
}
fn default_pdu_cache_capacity() -> u32 {
150_000
}
fn default_cleanup_second_interval() -> u32 {
1 * 60 // every minute
}
fn default_max_request_size() -> u32 {
20 * 1024 * 1024 // Default to 20 MB
}
fn default_max_concurrent_requests() -> u16 {
100
}
fn default_log() -> String {
"info,state_res=warn,rocket=off,_=off,sled=off".to_owned()
}
fn default_turn_ttl() -> u64 {
60 * 60 * 24
}

6
src/database/proxy.rs → src/config/proxy.rs
View file

@ -125,7 +125,7 @@ impl WildCardedDomain {
}
impl std::str::FromStr for WildCardedDomain {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
fn from_str(s: &str) -> Result<Self, Self::Err> {
// maybe do some domain validation?
Ok(if s.starts_with("*.") {
WildCardedDomain::WildCarded(s[1..].to_owned())
@ -136,8 +136,8 @@ impl std::str::FromStr for WildCardedDomain {
})
}
}
impl<'de> serde::de::Deserialize<'de> for WildCardedDomain {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
impl<'de> Deserialize<'de> for WildCardedDomain {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{

393
src/database.rs
View file

@ -6,7 +6,6 @@ pub mod appservice;
pub mod globals;
pub mod key_backups;
pub mod media;
pub mod proxy;
pub mod pusher;
pub mod rooms;
pub mod sending;
@ -14,7 +13,7 @@ pub mod transaction_ids;
pub mod uiaa;
pub mod users;
use crate::{utils, Error, Result};
use crate::{utils, Config, Error, Result};
use abstraction::DatabaseEngine;
use directories::ProjectDirs;
use lru_cache::LruCache;
@ -24,11 +23,9 @@ use rocket::{
request::{FromRequest, Request},
Shutdown, State,
};
use ruma::{DeviceId, EventId, RoomId, ServerName, UserId};
use serde::{de::IgnoredAny, Deserialize};
use ruma::{DeviceId, EventId, RoomId, UserId};
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
fs::{self, remove_dir_all},
io::Write,
mem::size_of,
@ -37,109 +34,12 @@ use std::{
sync::{Arc, Mutex, RwLock},
};
use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
use tracing::{debug, error, warn};
use tracing::{debug, error, info, warn};
use self::proxy::ProxyConfig;
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
server_name: Box<ServerName>,
database_path: String,
#[serde(default = "default_db_cache_capacity_mb")]
db_cache_capacity_mb: f64,
#[serde(default = "default_pdu_cache_capacity")]
pdu_cache_capacity: u32,
#[serde(default = "default_sqlite_wal_clean_second_interval")]
sqlite_wal_clean_second_interval: u32,
#[serde(default = "default_max_request_size")]
max_request_size: u32,
#[serde(default = "default_max_concurrent_requests")]
max_concurrent_requests: u16,
#[serde(default = "false_fn")]
allow_registration: bool,
#[serde(default = "true_fn")]
allow_encryption: bool,
#[serde(default = "false_fn")]
allow_federation: bool,
#[serde(default = "false_fn")]
pub allow_jaeger: bool,
#[serde(default = "false_fn")]
pub tracing_flame: bool,
#[serde(default)]
proxy: ProxyConfig,
jwt_secret: Option<String>,
#[serde(default = "Vec::new")]
trusted_servers: Vec<Box<ServerName>>,
#[serde(default = "default_log")]
pub log: String,
#[serde(flatten)]
catchall: BTreeMap<String, IgnoredAny>,
}
const DEPRECATED_KEYS: &[&str] = &["cache_capacity"];
impl Config {
pub fn warn_deprecated(&self) {
let mut was_deprecated = false;
for key in self
.catchall
.keys()
.filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
{
warn!("Config parameter {} is deprecated", key);
was_deprecated = true;
}
if was_deprecated {
warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
}
}
}
fn false_fn() -> bool {
false
}
fn true_fn() -> bool {
true
}
fn default_db_cache_capacity_mb() -> f64 {
200.0
}
fn default_pdu_cache_capacity() -> u32 {
100_000
}
fn default_sqlite_wal_clean_second_interval() -> u32 {
1 * 60 // every minute
}
fn default_max_request_size() -> u32 {
20 * 1024 * 1024 // Default to 20 MB
}
fn default_max_concurrent_requests() -> u16 {
100
}
fn default_log() -> String {
"info,state_res=warn,rocket=off,_=off,sled=off".to_owned()
}
#[cfg(feature = "sled")]
pub type Engine = abstraction::sled::Engine;
#[cfg(feature = "sqlite")]
pub type Engine = abstraction::sqlite::Engine;
#[cfg(feature = "heed")]
pub type Engine = abstraction::heed::Engine;
use self::admin::create_admin_room;
pub struct Database {
_db: Arc<Engine>,
_db: Arc<dyn DatabaseEngine>,
pub globals: globals::Globals,
pub users: users::Users,
pub uiaa: uiaa::Uiaa,
@ -167,28 +67,48 @@ impl Database {
Ok(())
}
fn check_sled_or_sqlite_db(config: &Config) -> Result<()> {
#[cfg(feature = "backend_sqlite")]
{
let path = Path::new(&config.database_path);
fn check_db_setup(config: &Config) -> Result<()> {
let path = Path::new(&config.database_path);
let sled_exists = path.join("db").exists();
let sqlite_exists = path.join("conduit.db").exists();
if sled_exists {
if sqlite_exists {
// most likely an in-place directory, only warn
warn!("Both sled and sqlite databases are detected in database directory");
warn!("Currently running from the sqlite database, but consider removing sled database files to free up space")
} else {
error!(
"Sled database detected, conduit now uses sqlite for database operations"
);
error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite");
return Err(Error::bad_config(
"sled database detected, migrate to sqlite",
));
}
}
let sled_exists = path.join("db").exists();
let sqlite_exists = path.join("conduit.db").exists();
let rocksdb_exists = path.join("IDENTITY").exists();
let mut count = 0;
if sled_exists {
count += 1;
}
if sqlite_exists {
count += 1;
}
if rocksdb_exists {
count += 1;
}
if count > 1 {
warn!("Multiple databases at database_path detected");
return Ok(());
}
if sled_exists && config.database_backend != "sled" {
return Err(Error::bad_config(
"Found sled at database_path, but is not specified in config.",
));
}
if sqlite_exists && config.database_backend != "sqlite" {
return Err(Error::bad_config(
"Found sqlite at database_path, but is not specified in config.",
));
}
if rocksdb_exists && config.database_backend != "rocksdb" {
return Err(Error::bad_config(
"Found rocksdb at database_path, but is not specified in config.",
));
}
Ok(())
@ -196,9 +116,36 @@ impl Database {
/// Load an existing database or create a new one.
pub async fn load_or_create(config: &Config) -> Result<Arc<TokioRwLock<Self>>> {
Self::check_sled_or_sqlite_db(&config)?;
Self::check_db_setup(config)?;
let builder = Engine::open(&config)?;
if !Path::new(&config.database_path).exists() {
std::fs::create_dir_all(&config.database_path)
.map_err(|_| Error::BadConfig("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?;
}
let builder: Arc<dyn DatabaseEngine> = match &*config.database_backend {
"sqlite" => {
#[cfg(not(feature = "sqlite"))]
return Err(Error::BadConfig("Database backend not found."));
#[cfg(feature = "sqlite")]
Arc::new(Arc::<abstraction::sqlite::Engine>::open(config)?)
}
"rocksdb" => {
#[cfg(not(feature = "rocksdb"))]
return Err(Error::BadConfig("Database backend not found."));
#[cfg(feature = "rocksdb")]
Arc::new(Arc::<abstraction::rocksdb::Engine>::open(config)?)
}
"persy" => {
#[cfg(not(feature = "persy"))]
return Err(Error::BadConfig("Database backend not found."));
#[cfg(feature = "persy")]
Arc::new(Arc::<abstraction::persy::Engine>::open(config)?)
}
_ => {
return Err(Error::BadConfig("Database backend not found."));
}
};
if config.max_request_size < 1024 {
eprintln!("ERROR: Max request size is less than 1KB. Please increase it.");
@ -225,12 +172,12 @@ impl Database {
userid_masterkeyid: builder.open_tree("userid_masterkeyid")?,
userid_selfsigningkeyid: builder.open_tree("userid_selfsigningkeyid")?,
userid_usersigningkeyid: builder.open_tree("userid_usersigningkeyid")?,
userfilterid_filter: builder.open_tree("userfilterid_filter")?,
todeviceid_events: builder.open_tree("todeviceid_events")?,
},
uiaa: uiaa::Uiaa {
userdevicesessionid_uiaainfo: builder.open_tree("userdevicesessionid_uiaainfo")?,
userdevicesessionid_uiaarequest: builder
.open_tree("userdevicesessionid_uiaarequest")?,
userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()),
},
rooms: rooms::Rooms {
edus: rooms::RoomEdus {
@ -265,6 +212,8 @@ impl Database {
userroomid_leftstate: builder.open_tree("userroomid_leftstate")?,
roomuserid_leftcount: builder.open_tree("roomuserid_leftcount")?,
lazyloadedids: builder.open_tree("lazyloadedids")?,
userroomid_notificationcount: builder.open_tree("userroomid_notificationcount")?,
userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?,
@ -293,14 +242,27 @@ impl Database {
.try_into()
.expect("pdu cache capacity fits into usize"),
)),
auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
auth_chain_cache: Mutex::new(LruCache::new(
(100_000.0 * config.conduit_cache_capacity_modifier) as usize,
)),
shorteventid_cache: Mutex::new(LruCache::new(
(100_000.0 * config.conduit_cache_capacity_modifier) as usize,
)),
eventidshort_cache: Mutex::new(LruCache::new(
(100_000.0 * config.conduit_cache_capacity_modifier) as usize,
)),
shortstatekey_cache: Mutex::new(LruCache::new(
(100_000.0 * config.conduit_cache_capacity_modifier) as usize,
)),
statekeyshort_cache: Mutex::new(LruCache::new(
(100_000.0 * config.conduit_cache_capacity_modifier) as usize,
)),
our_real_users_cache: RwLock::new(HashMap::new()),
appservice_in_room_cache: RwLock::new(HashMap::new()),
stateinfo_cache: Mutex::new(LruCache::new(1000)),
lazy_load_waiting: Mutex::new(HashMap::new()),
stateinfo_cache: Mutex::new(LruCache::new(
(100.0 * config.conduit_cache_capacity_modifier) as usize,
)),
},
account_data: account_data::AccountData {
roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
@ -341,10 +303,32 @@ impl Database {
)?,
}));
{
let db = db.read().await;
let guard = db.read().await;
// Matrix resource ownership is based on the server name; changing it
// requires recreating the database from scratch.
if guard.users.count()? > 0 {
let conduit_user =
UserId::parse_with_server_name("conduit", guard.globals.server_name())
.expect("@conduit:server_name is valid");
if !guard.users.exists(&conduit_user)? {
error!(
"The {} server user does not exist, and the database is not new.",
conduit_user
);
return Err(Error::bad_database(
"Cannot reuse an existing database after changing the server name, please delete the old one first."
));
}
}
// If the database has any data, perform data migrations before starting
let latest_database_version = 11;
if guard.users.count()? > 0 {
let db = &*guard;
// MIGRATIONS
// TODO: database versions of new dbs should probably not be 0
if db.globals.database_version()? < 1 {
for (roomserverid, _) in db.rooms.roomserverids.iter() {
let mut parts = roomserverid.split(|&b| b == 0xff);
@ -365,7 +349,7 @@ impl Database {
db.globals.bump_database_version(1)?;
println!("Migration: 0 -> 1 finished");
warn!("Migration: 0 -> 1 finished");
}
if db.globals.database_version()? < 2 {
@ -384,7 +368,7 @@ impl Database {
db.globals.bump_database_version(2)?;
println!("Migration: 1 -> 2 finished");
warn!("Migration: 1 -> 2 finished");
}
if db.globals.database_version()? < 3 {
@ -402,7 +386,7 @@ impl Database {
db.globals.bump_database_version(3)?;
println!("Migration: 2 -> 3 finished");
warn!("Migration: 2 -> 3 finished");
}
if db.globals.database_version()? < 4 {
@ -425,7 +409,7 @@ impl Database {
db.globals.bump_database_version(4)?;
println!("Migration: 3 -> 4 finished");
warn!("Migration: 3 -> 4 finished");
}
if db.globals.database_version()? < 5 {
@ -449,26 +433,25 @@ impl Database {
db.globals.bump_database_version(5)?;
println!("Migration: 4 -> 5 finished");
warn!("Migration: 4 -> 5 finished");
}
if db.globals.database_version()? < 6 {
// Set room member count
for (roomid, _) in db.rooms.roomid_shortstatehash.iter() {
let room_id =
RoomId::try_from(utils::string_from_bytes(&roomid).unwrap()).unwrap();
db.rooms.update_joined_count(&room_id, &db)?;
let string = utils::string_from_bytes(&roomid).unwrap();
let room_id = <&RoomId>::try_from(string.as_str()).unwrap();
db.rooms.update_joined_count(room_id, &db)?;
}
db.globals.bump_database_version(6)?;
println!("Migration: 5 -> 6 finished");
warn!("Migration: 5 -> 6 finished");
}
if db.globals.database_version()? < 7 {
// Upgrade state store
let mut last_roomstates: HashMap<RoomId, u64> = HashMap::new();
let mut last_roomstates: HashMap<Box<RoomId>, u64> = HashMap::new();
let mut current_sstatehash: Option<u64> = None;
let mut current_room = None;
let mut current_state = HashSet::new();
@ -494,13 +477,13 @@ impl Database {
if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew = current_state
.difference(&parent_stateinfo.1)
.cloned()
.copied()
.collect::<HashSet<_>>();
let statediffremoved = parent_stateinfo
.1
.difference(&current_state)
.cloned()
.copied()
.collect::<HashSet<_>>();
(statediffnew, statediffremoved)
@ -549,7 +532,7 @@ impl Database {
if let Some(current_sstatehash) = current_sstatehash {
handle_state(
current_sstatehash,
current_room.as_ref().unwrap(),
current_room.as_deref().unwrap(),
current_state,
&mut last_roomstates,
)?;
@ -565,10 +548,9 @@ impl Database {
.get(&seventid)
.unwrap()
.unwrap();
let event_id =
EventId::try_from(utils::string_from_bytes(&event_id).unwrap())
.unwrap();
let pdu = db.rooms.get_pdu(&event_id).unwrap().unwrap();
let string = utils::string_from_bytes(&event_id).unwrap();
let event_id = <&EventId>::try_from(string.as_str()).unwrap();
let pdu = db.rooms.get_pdu(event_id).unwrap().unwrap();
if Some(&pdu.room_id) != current_room.as_ref() {
current_room = Some(pdu.room_id.clone());
@ -583,7 +565,7 @@ impl Database {
if let Some(current_sstatehash) = current_sstatehash {
handle_state(
current_sstatehash,
current_room.as_ref().unwrap(),
current_room.as_deref().unwrap(),
current_state,
&mut last_roomstates,
)?;
@ -591,7 +573,7 @@ impl Database {
db.globals.bump_database_version(7)?;
println!("Migration: 6 -> 7 finished");
warn!("Migration: 6 -> 7 finished");
}
if db.globals.database_version()? < 8 {
@ -599,7 +581,7 @@ impl Database {
for (room_id, _) in db.rooms.roomid_shortstatehash.iter() {
let shortroomid = db.globals.next_count()?.to_be_bytes();
db.rooms.roomid_shortroomid.insert(&room_id, &shortroomid)?;
println!("Migration: 8");
info!("Migration: 8");
}
// Update pduids db layout
let mut batch = db.rooms.pduid_pdu.iter().filter_map(|(key, v)| {
@ -613,7 +595,7 @@ impl Database {
let short_room_id = db
.rooms
.roomid_shortroomid
.get(&room_id)
.get(room_id)
.unwrap()
.expect("shortroomid should exist");
@ -636,7 +618,7 @@ impl Database {
let short_room_id = db
.rooms
.roomid_shortroomid
.get(&room_id)
.get(room_id)
.unwrap()
.expect("shortroomid should exist");
@ -650,7 +632,7 @@ impl Database {
db.globals.bump_database_version(8)?;
println!("Migration: 7 -> 8 finished");
warn!("Migration: 7 -> 8 finished");
}
if db.globals.database_version()? < 9 {
@ -672,7 +654,7 @@ impl Database {
let short_room_id = db
.rooms
.roomid_shortroomid
.get(&room_id)
.get(room_id)
.unwrap()
.expect("shortroomid should exist");
let mut new_key = short_room_id;
@ -692,9 +674,9 @@ impl Database {
println!("smaller batch done");
}
println!("Deleting starts");
info!("Deleting starts");
let batch2 = db
let batch2: Vec<_> = db
.rooms
.tokenids
.iter()
@ -706,7 +688,7 @@ impl Database {
None
}
})
.collect::<Vec<_>>();
.collect();
for key in batch2 {
println!("del");
@ -715,7 +697,7 @@ impl Database {
db.globals.bump_database_version(9)?;
println!("Migration: 8 -> 9 finished");
warn!("Migration: 8 -> 9 finished");
}
if db.globals.database_version()? < 10 {
@ -734,11 +716,37 @@ impl Database {
db.globals.bump_database_version(10)?;
println!("Migration: 9 -> 10 finished");
warn!("Migration: 9 -> 10 finished");
}
}
let guard = db.read().await;
if db.globals.database_version()? < 11 {
db._db
.open_tree("userdevicesessionid_uiaarequest")?
.clear()?;
db.globals.bump_database_version(11)?;
warn!("Migration: 10 -> 11 finished");
}
assert_eq!(11, latest_database_version);
info!(
"Loaded {} database with version {}",
config.database_backend, latest_database_version
);
} else {
guard
.globals
.bump_database_version(latest_database_version)?;
// Create the admin room and server user on first run
create_admin_room(&guard).await?;
warn!(
"Created new {} database with version {}",
config.database_backend, latest_database_version
);
}
// This data is probably outdated
guard.rooms.edus.presenceid_presence.clear()?;
@ -750,18 +758,13 @@ impl Database {
drop(guard);
#[cfg(feature = "sqlite")]
{
Self::start_wal_clean_task(Arc::clone(&db), &config).await;
}
Self::start_cleanup_task(Arc::clone(&db), config).await;
Ok(db)
}
#[cfg(feature = "conduit_bin")]
pub async fn start_on_shutdown_tasks(db: Arc<TokioRwLock<Self>>, shutdown: Shutdown) {
use tracing::info;
tokio::spawn(async move {
shutdown.await;
@ -810,12 +813,21 @@ impl Database {
// Events for rooms we are in
for room_id in self.rooms.rooms_joined(user_id).filter_map(|r| r.ok()) {
let short_roomid = self
.rooms
.get_shortroomid(&room_id)
.ok()
.flatten()
.expect("room exists")
.to_be_bytes()
.to_vec();
let roomid_bytes = room_id.as_bytes().to_vec();
let mut roomid_prefix = roomid_bytes.clone();
roomid_prefix.push(0xff);
// PDUs
futures.push(self.rooms.pduid_pdu.watch_prefix(&roomid_prefix));
futures.push(self.rooms.pduid_pdu.watch_prefix(&short_roomid));
// EDUs
futures.push(
@ -882,15 +894,8 @@ impl Database {
res
}
#[cfg(feature = "sqlite")]
#[tracing::instrument(skip(self))]
pub fn flush_wal(&self) -> Result<()> {
self._db.flush_wal()
}
#[cfg(feature = "sqlite")]
#[tracing::instrument(skip(db, config))]
pub async fn start_wal_clean_task(db: Arc<TokioRwLock<Self>>, config: &Config) {
pub async fn start_cleanup_task(db: Arc<TokioRwLock<Self>>, config: &Config) {
use tokio::time::interval;
#[cfg(unix)]
@ -899,7 +904,7 @@ impl Database {
use std::time::{Duration, Instant};
let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64);
let timer_interval = Duration::from_secs(config.cleanup_second_interval as u64);
tokio::spawn(async move {
let mut i = interval(timer_interval);
@ -910,23 +915,23 @@ impl Database {
#[cfg(unix)]
tokio::select! {
_ = i.tick() => {
info!("wal-trunc: Timer ticked");
info!("cleanup: Timer ticked");
}
_ = s.recv() => {
info!("wal-trunc: Received SIGHUP");
info!("cleanup: Received SIGHUP");
}
};
#[cfg(not(unix))]
{
i.tick().await;
info!("wal-trunc: Timer ticked")
info!("cleanup: Timer ticked")
}
let start = Instant::now();
if let Err(e) = db.read().await.flush_wal() {
error!("wal-trunc: Errored: {}", e);
if let Err(e) = db.read().await._db.cleanup() {
error!("cleanup: Errored: {}", e);
} else {
info!("wal-trunc: Flushed in {:?}", start.elapsed());
info!("cleanup: Finished in {:?}", start.elapsed());
}
}
});
@ -950,7 +955,7 @@ impl<'r> FromRequest<'r> for DatabaseGuard {
async fn from_request(req: &'r Request<'_>) -> rocket::request::Outcome<Self, ()> {
let db = try_outcome!(req.guard::<&State<Arc<TokioRwLock<Database>>>>().await);
Ok(DatabaseGuard(Arc::clone(&db).read_owned().await)).or_forward(())
Ok(DatabaseGuard(Arc::clone(db).read_owned().await)).or_forward(())
}
}

34
src/database/abstraction.rs
View file

@ -12,17 +12,39 @@ pub mod sqlite;
#[cfg(feature = "heed")]
pub mod heed;
pub trait DatabaseEngine: Sized {
fn open(config: &Config) -> Result<Arc<Self>>;
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;
fn flush(self: &Arc<Self>) -> Result<()>;
#[cfg(feature = "rocksdb")]
pub mod rocksdb;
#[cfg(feature = "persy")]
pub mod persy;
#[cfg(any(
feature = "sqlite",
feature = "rocksdb",
feature = "heed",
feature = "persy"
))]
pub mod watchers;
pub trait DatabaseEngine: Send + Sync {
fn open(config: &Config) -> Result<Self>
where
Self: Sized;
fn open_tree(&self, name: &'static str) -> Result<Arc<dyn Tree>>;
fn flush(&self) -> Result<()>;
fn cleanup(&self) -> Result<()> {
Ok(())
}
fn memory_usage(&self) -> Result<String> {
Ok("Current database engine does not support memory usage reporting.".to_owned())
}
}
pub trait Tree: Send + Sync {
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>>;
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>;
fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()>;
fn insert_batch(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()>;
fn remove(&self, key: &[u8]) -> Result<()>;
@ -35,7 +57,7 @@ pub trait Tree: Send + Sync {
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;
fn increment_batch(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;
fn scan_prefix<'a>(
&'a self,

48
src/database/abstraction/heed.rs
View file

@ -1,15 +1,13 @@
use super::super::Config;
use super::{super::Config, watchers::Watchers};
use crossbeam::channel::{bounded, Sender as ChannelSender};
use threadpool::ThreadPool;
use crate::{Error, Result};
use std::{
collections::HashMap,
future::Future,
pin::Pin,
sync::{Arc, Mutex, RwLock},
sync::{Arc, Mutex},
};
use tokio::sync::oneshot::Sender;
use super::{DatabaseEngine, Tree};
@ -23,7 +21,7 @@ pub struct Engine {
pub struct EngineTree {
engine: Arc<Engine>,
tree: Arc<heed::UntypedDatabase>,
watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
watchers: Watchers,
}
fn convert_error(error: heed::Error) -> Error {
@ -60,7 +58,7 @@ impl DatabaseEngine for Engine {
.create_database(Some(name))
.map_err(convert_error)?,
),
watchers: RwLock::new(HashMap::new()),
watchers: Default::default(),
}))
}
@ -145,29 +143,7 @@ impl Tree for EngineTree {
.put(&mut txn, &key, &value)
.map_err(convert_error)?;
txn.commit().map_err(convert_error)?;
let watchers = self.watchers.read().unwrap();
let mut triggered = Vec::new();
for length in 0..=key.len() {
if watchers.contains_key(&key[..length]) {
triggered.push(&key[..length]);
}
}
drop(watchers);
if !triggered.is_empty() {
let mut watchers = self.watchers.write().unwrap();
for prefix in triggered {
if let Some(txs) = watchers.remove(prefix) {
for tx in txs {
let _ = tx.send(());
}
}
}
};
self.watchers.wake(key);
Ok(())
}
@ -223,18 +199,6 @@ impl Tree for EngineTree {
#[tracing::instrument(skip(self, prefix))]
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let (tx, rx) = tokio::sync::oneshot::channel();
self.watchers
.write()
.unwrap()
.entry(prefix.to_vec())
.or_default()
.push(tx);
Box::pin(async move {
// Tx is never destroyed
rx.await.unwrap();
})
self.watchers.watch(prefix)
}
}

207
src/database/abstraction/persy.rs Normal file
View file

@ -0,0 +1,207 @@
use crate::{
database::{
abstraction::{watchers::Watchers, DatabaseEngine, Tree},
Config,
},
Result,
};
use persy::{ByteVec, OpenOptions, Persy, Transaction, TransactionConfig, ValueMode};
use std::{future::Future, pin::Pin, sync::Arc};
use tracing::warn;
pub struct Engine {
persy: Persy,
}
impl DatabaseEngine for Arc<Engine> {
fn open(config: &Config) -> Result<Self> {
let mut cfg = persy::Config::new();
cfg.change_cache_size((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64);
let persy = OpenOptions::new()
.create(true)
.config(cfg)
.open(&format!("{}/db.persy", config.database_path))?;
Ok(Arc::new(Engine { persy }))
}
fn open_tree(&self, name: &'static str) -> Result<Arc<dyn Tree>> {
// Create if it doesn't exist
if !self.persy.exists_index(name)? {
let mut tx = self.persy.begin()?;
tx.create_index::<ByteVec, ByteVec>(name, ValueMode::Replace)?;
tx.prepare()?.commit()?;
}
Ok(Arc::new(PersyTree {
persy: self.persy.clone(),
name: name.to_owned(),
watchers: Watchers::default(),
}))
}
fn flush(&self) -> Result<()> {
Ok(())
}
}
pub struct PersyTree {
persy: Persy,
name: String,
watchers: Watchers,
}
impl PersyTree {
fn begin(&self) -> Result<Transaction> {
Ok(self
.persy
.begin_with(TransactionConfig::new().set_background_sync(true))?)
}
}
impl Tree for PersyTree {
#[tracing::instrument(skip(self, key))]
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
let result = self
.persy
.get::<ByteVec, ByteVec>(&self.name, &ByteVec::from(key))?
.next()
.map(|v| (*v).to_owned());
Ok(result)
}
#[tracing::instrument(skip(self, key, value))]
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
self.insert_batch(&mut Some((key.to_owned(), value.to_owned())).into_iter())?;
self.watchers.wake(key);
Ok(())
}
#[tracing::instrument(skip(self, iter))]
fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
let mut tx = self.begin()?;
for (key, value) in iter {
tx.put::<ByteVec, ByteVec>(
&self.name,
ByteVec::from(key.clone()),
ByteVec::from(value),
)?;
}
tx.prepare()?.commit()?;
Ok(())
}
#[tracing::instrument(skip(self, iter))]
fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
let mut tx = self.begin()?;
for key in iter {
let old = tx
.get::<ByteVec, ByteVec>(&self.name, &ByteVec::from(key.clone()))?
.next()
.map(|v| (*v).to_owned());
let new = crate::utils::increment(old.as_deref()).unwrap();
tx.put::<ByteVec, ByteVec>(&self.name, ByteVec::from(key), ByteVec::from(new))?;
}
tx.prepare()?.commit()?;
Ok(())
}
#[tracing::instrument(skip(self, key))]
fn remove(&self, key: &[u8]) -> Result<()> {
let mut tx = self.begin()?;
tx.remove::<ByteVec, ByteVec>(&self.name, ByteVec::from(key), None)?;
tx.prepare()?.commit()?;
Ok(())
}
#[tracing::instrument(skip(self))]
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let iter = self.persy.range::<ByteVec, ByteVec, _>(&self.name, ..);
match iter {
Ok(iter) => Box::new(iter.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
})),
Err(e) => {
warn!("error iterating {:?}", e);
Box::new(std::iter::empty())
}
}
}
#[tracing::instrument(skip(self, from, backwards))]
fn iter_from<'a>(
&'a self,
from: &[u8],
backwards: bool,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let range = if backwards {
self.persy
.range::<ByteVec, ByteVec, _>(&self.name, ..=ByteVec::from(from))
} else {
self.persy
.range::<ByteVec, ByteVec, _>(&self.name, ByteVec::from(from)..)
};
match range {
Ok(iter) => {
let map = iter.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
});
if backwards {
Box::new(map.rev())
} else {
Box::new(map)
}
}
Err(e) => {
warn!("error iterating with prefix {:?}", e);
Box::new(std::iter::empty())
}
}
}
#[tracing::instrument(skip(self, key))]
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
self.increment_batch(&mut Some(key.to_owned()).into_iter())?;
Ok(self.get(key)?.unwrap())
}
#[tracing::instrument(skip(self, prefix))]
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
let range_prefix = ByteVec::from(prefix.clone());
let range = self
.persy
.range::<ByteVec, ByteVec, _>(&self.name, range_prefix..);
match range {
Ok(iter) => {
let owned_prefix = prefix.clone();
Box::new(
iter.take_while(move |(k, _)| (*k).starts_with(&owned_prefix))
.filter_map(|(k, v)| {
v.into_iter()
.map(|val| ((*k).to_owned().into(), (*val).to_owned().into()))
.next()
}),
)
}
Err(e) => {
warn!("error scanning prefix {:?}", e);
Box::new(std::iter::empty())
}
}
}
#[tracing::instrument(skip(self, prefix))]
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
self.watchers.watch(prefix)
}
}

231
src/database/abstraction/rocksdb.rs Normal file
View file

@ -0,0 +1,231 @@
use super::{super::Config, watchers::Watchers, DatabaseEngine, Tree};
use crate::{utils, Result};
use std::{future::Future, pin::Pin, sync::Arc, sync::RwLock};
pub struct Engine {
rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>,
max_open_files: i32,
cache: rocksdb::Cache,
old_cfs: Vec<String>,
}
pub struct RocksDbEngineTree<'a> {
db: Arc<Engine>,
name: &'a str,
watchers: Watchers,
write_lock: RwLock<()>,
}
fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options {
let mut block_based_options = rocksdb::BlockBasedOptions::default();
block_based_options.set_block_cache(rocksdb_cache);
// "Difference of spinning disk"
// https://zhangyuchi.gitbooks.io/rocksdbbook/content/RocksDB-Tuning-Guide.html
block_based_options.set_block_size(4 * 1024);
block_based_options.set_cache_index_and_filter_blocks(true);
let mut db_opts = rocksdb::Options::default();
db_opts.set_block_based_table_factory(&block_based_options);
db_opts.set_optimize_filters_for_hits(true);
db_opts.set_skip_stats_update_on_db_open(true);
db_opts.set_level_compaction_dynamic_level_bytes(true);
db_opts.set_target_file_size_base(256 * 1024 * 1024);
//db_opts.set_compaction_readahead_size(2 * 1024 * 1024);
//db_opts.set_use_direct_reads(true);
//db_opts.set_use_direct_io_for_flush_and_compaction(true);
db_opts.create_if_missing(true);
db_opts.increase_parallelism(num_cpus::get() as i32);
db_opts.set_max_open_files(max_open_files);
db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.optimize_level_style_compaction(10 * 1024 * 1024);
let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1);
db_opts.set_prefix_extractor(prefix_extractor);
db_opts
}
impl DatabaseEngine for Arc<Engine> {
fn open(config: &Config) -> Result<Self> {
let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap();
let db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache);
let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf(
&db_opts,
&config.database_path,
)
.unwrap_or_default();
let db = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::open_cf_descriptors(
&db_opts,
&config.database_path,
cfs.iter().map(|name| {
rocksdb::ColumnFamilyDescriptor::new(
name,
db_options(config.rocksdb_max_open_files, &rocksdb_cache),
)
}),
)?;
Ok(Arc::new(Engine {
rocks: db,
max_open_files: config.rocksdb_max_open_files,
cache: rocksdb_cache,
old_cfs: cfs,
}))
}
fn open_tree(&self, name: &'static str) -> Result<Arc<dyn Tree>> {
if !self.old_cfs.contains(&name.to_owned()) {
// Create if it didn't exist
let _ = self
.rocks
.create_cf(name, &db_options(self.max_open_files, &self.cache));
}
Ok(Arc::new(RocksDbEngineTree {
name,
db: Arc::clone(self),
watchers: Watchers::default(),
write_lock: RwLock::new(()),
}))
}
fn flush(&self) -> Result<()> {
// TODO?
Ok(())
}
fn memory_usage(&self) -> Result<String> {
let stats =
rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?;
Ok(format!(
"Approximate memory usage of all the mem-tables: {:.3} MB\n\
Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\
Approximate memory usage of all the table readers: {:.3} MB\n\
Approximate memory usage by cache: {:.3} MB\n\
Approximate memory usage by cache pinned: {:.3} MB\n\
",
stats.mem_table_total as f64 / 1024.0 / 1024.0,
stats.mem_table_unflushed as f64 / 1024.0 / 1024.0,
stats.mem_table_readers_total as f64 / 1024.0 / 1024.0,
stats.cache_total as f64 / 1024.0 / 1024.0,
self.cache.get_pinned_usage() as f64 / 1024.0 / 1024.0,
))
}
}
impl RocksDbEngineTree<'_> {
fn cf(&self) -> Arc<rocksdb::BoundColumnFamily<'_>> {
self.db.rocks.cf_handle(self.name).unwrap()
}
}
impl Tree for RocksDbEngineTree<'_> {
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
Ok(self.db.rocks.get_cf(&self.cf(), key)?)
}
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
let lock = self.write_lock.read().unwrap();
self.db.rocks.put_cf(&self.cf(), key, value)?;
drop(lock);
self.watchers.wake(key);
Ok(())
}
fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
for (key, value) in iter {
self.db.rocks.put_cf(&self.cf(), key, value)?;
}
Ok(())
}
fn remove(&self, key: &[u8]) -> Result<()> {
Ok(self.db.rocks.delete_cf(&self.cf(), key)?)
}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
Box::new(
self.db
.rocks
.iterator_cf(&self.cf(), rocksdb::IteratorMode::Start)
.map(|(k, v)| (Vec::from(k), Vec::from(v))),
)
}
fn iter_from<'a>(
&'a self,
from: &[u8],
backwards: bool,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
Box::new(
self.db
.rocks
.iterator_cf(
&self.cf(),
rocksdb::IteratorMode::From(
from,
if backwards {
rocksdb::Direction::Reverse
} else {
rocksdb::Direction::Forward
},
),
)
.map(|(k, v)| (Vec::from(k), Vec::from(v))),
)
}
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
let lock = self.write_lock.write().unwrap();
let old = self.db.rocks.get_cf(&self.cf(), &key)?;
let new = utils::increment(old.as_deref()).unwrap();
self.db.rocks.put_cf(&self.cf(), key, &new)?;
drop(lock);
Ok(new)
}
fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
let lock = self.write_lock.write().unwrap();
for key in iter {
let old = self.db.rocks.get_cf(&self.cf(), &key)?;
let new = utils::increment(old.as_deref()).unwrap();
self.db.rocks.put_cf(&self.cf(), key, new)?;
}
drop(lock);
Ok(())
}
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
Box::new(
self.db
.rocks
.iterator_cf(
&self.cf(),
rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward),
)
.map(|(k, v)| (Vec::from(k), Vec::from(v)))
.take_while(move |(k, _)| k.starts_with(&prefix)),
)
}
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
self.watchers.watch(prefix)
}
}

69
src/database/abstraction/sqlite.rs
View file

@ -1,17 +1,15 @@
use super::{DatabaseEngine, Tree};
use super::{watchers::Watchers, DatabaseEngine, Tree};
use crate::{database::Config, Result};
use parking_lot::{Mutex, MutexGuard, RwLock};
use parking_lot::{Mutex, MutexGuard};
use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
use std::{
cell::RefCell,
collections::HashMap,
future::Future,
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
};
use thread_local::ThreadLocal;
use tokio::sync::oneshot::Sender;
use tracing::debug;
thread_local! {
@ -56,7 +54,7 @@ impl Engine {
conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
conn.pragma_update(Some(Main), "wal_autocheckpoint", &2000)?;
conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?;
Ok(conn)
}
@ -77,13 +75,13 @@ impl Engine {
pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
self.write_lock()
.pragma_update(Some(Main), "wal_checkpoint", &"TRUNCATE")?;
.pragma_update(Some(Main), "wal_checkpoint", &"RESTART")?;
Ok(())
}
}
impl DatabaseEngine for Engine {
fn open(config: &Config) -> Result<Arc<Self>> {
impl DatabaseEngine for Arc<Engine> {
fn open(config: &Config) -> Result<Self> {
let path = Path::new(&config.database_path).join("conduit.db");
// calculates cache-size per permanent connection
@ -94,7 +92,7 @@ impl DatabaseEngine for Engine {
/ ((num_cpus::get().max(1) * 2) + 1) as f64)
as u32;
let writer = Mutex::new(Self::prepare_conn(&path, cache_size_per_thread)?);
let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?);
let arc = Arc::new(Engine {
writer,
@ -107,26 +105,30 @@ impl DatabaseEngine for Engine {
Ok(arc)
}
fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> {
fn open_tree(&self, name: &str) -> Result<Arc<dyn Tree>> {
self.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name), [])?;
Ok(Arc::new(SqliteTable {
engine: Arc::clone(self),
name: name.to_owned(),
watchers: RwLock::new(HashMap::new()),
watchers: Watchers::default(),
}))
}
fn flush(self: &Arc<Self>) -> Result<()> {
fn flush(&self) -> Result<()> {
// we enabled PRAGMA synchronous=normal, so this should not be necessary
Ok(())
}
fn cleanup(&self) -> Result<()> {
self.flush_wal()
}
}
pub struct SqliteTable {
engine: Arc<Engine>,
name: String,
watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
watchers: Watchers,
}
type TupleOfBytes = (Vec<u8>, Vec<u8>);
@ -192,7 +194,7 @@ impl SqliteTable {
impl Tree for SqliteTable {
#[tracing::instrument(skip(self, key))]
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
self.get_with_guard(&self.engine.read_lock(), key)
self.get_with_guard(self.engine.read_lock(), key)
}
#[tracing::instrument(skip(self, key, value))]
@ -200,29 +202,7 @@ impl Tree for SqliteTable {
let guard = self.engine.write_lock();
self.insert_with_guard(&guard, key, value)?;
drop(guard);
let watchers = self.watchers.read();
let mut triggered = Vec::new();
for length in 0..=key.len() {
if watchers.contains_key(&key[..length]) {
triggered.push(&key[..length]);
}
}
drop(watchers);
if !triggered.is_empty() {
let mut watchers = self.watchers.write();
for prefix in triggered {
if let Some(txs) = watchers.remove(prefix) {
for tx in txs {
let _ = tx.send(());
}
}
}
};
self.watchers.wake(key);
Ok(())
}
@ -275,7 +255,7 @@ impl Tree for SqliteTable {
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
let guard = self.engine.read_lock_iterator();
self.iter_with_guard(&guard)
self.iter_with_guard(guard)
}
#[tracing::instrument(skip(self, from, backwards))]
@ -367,18 +347,7 @@ impl Tree for SqliteTable {
#[tracing::instrument(skip(self, prefix))]
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let (tx, rx) = tokio::sync::oneshot::channel();
self.watchers
.write()
.entry(prefix.to_vec())
.or_default()
.push(tx);
Box::pin(async move {
// Tx is never destroyed
rx.await.unwrap();
})
self.watchers.watch(prefix)
}
#[tracing::instrument(skip(self))]

54
src/database/abstraction/watchers.rs Normal file
View file

@ -0,0 +1,54 @@
use std::{
collections::{hash_map, HashMap},
future::Future,
pin::Pin,
sync::RwLock,
};
use tokio::sync::watch;
#[derive(Default)]
pub(super) struct Watchers {
watchers: RwLock<HashMap<Vec<u8>, (watch::Sender<()>, watch::Receiver<()>)>>,
}
impl Watchers {
pub(super) fn watch<'a>(
&'a self,
prefix: &[u8],
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec()) {
hash_map::Entry::Occupied(o) => o.get().1.clone(),
hash_map::Entry::Vacant(v) => {
let (tx, rx) = tokio::sync::watch::channel(());
v.insert((tx, rx.clone()));
rx
}
};
Box::pin(async move {
// Tx is never destroyed
rx.changed().await.unwrap();
})
}
pub(super) fn wake(&self, key: &[u8]) {
let watchers = self.watchers.read().unwrap();
let mut triggered = Vec::new();
for length in 0..=key.len() {
if watchers.contains_key(&key[..length]) {
triggered.push(&key[..length]);
}
}
drop(watchers);
if !triggered.is_empty() {
let mut watchers = self.watchers.write().unwrap();
for prefix in triggered {
if let Some(tx) = watchers.remove(prefix) {
let _ = tx.0.send(());
}
}
};
}
}

10
src/database/account_data.rs
View file

@ -6,7 +6,7 @@ use ruma::{
RoomId, UserId,
};
use serde::{de::DeserializeOwned, Serialize};
use std::{collections::HashMap, convert::TryFrom, sync::Arc};
use std::{collections::HashMap, sync::Arc};
use super::abstraction::Tree;
@ -32,13 +32,13 @@ impl AccountData {
.as_bytes()
.to_vec();
prefix.push(0xff);
prefix.extend_from_slice(&user_id.as_bytes());
prefix.extend_from_slice(user_id.as_bytes());
prefix.push(0xff);
let mut roomuserdataid = prefix.clone();
roomuserdataid.extend_from_slice(&globals.next_count()?.to_be_bytes());
roomuserdataid.push(0xff);
roomuserdataid.extend_from_slice(&event_type.as_bytes());
roomuserdataid.extend_from_slice(event_type.as_bytes());
let mut key = prefix;
key.extend_from_slice(event_type.as_bytes());
@ -83,7 +83,7 @@ impl AccountData {
.as_bytes()
.to_vec();
key.push(0xff);
key.extend_from_slice(&user_id.as_bytes());
key.extend_from_slice(user_id.as_bytes());
key.push(0xff);
key.extend_from_slice(kind.as_ref().as_bytes());
@ -118,7 +118,7 @@ impl AccountData {
.as_bytes()
.to_vec();
prefix.push(0xff);
prefix.extend_from_slice(&user_id.as_bytes());
prefix.extend_from_slice(user_id.as_bytes());
prefix.push(0xff);
// Skip the data that's exactly at since, because we sent that last time

802
src/database/admin.rs
View file

@ -1,33 +1,49 @@
use std::{
convert::{TryFrom, TryInto},
sync::Arc,
};
use std::{collections::BTreeMap, convert::TryFrom, convert::TryInto, sync::Arc, time::Instant};
use crate::{pdu::PduBuilder, Database};
use rocket::futures::{channel::mpsc, stream::StreamExt};
use crate::{
error::{Error, Result},
pdu::PduBuilder,
server_server, Database, PduEvent,
};
use clap::Parser;
use regex::Regex;
use rocket::{
futures::{channel::mpsc, stream::StreamExt},
http::RawStr,
};
use ruma::{
events::{room::message, EventType},
UserId,
events::room::{
canonical_alias::RoomCanonicalAliasEventContent,
create::RoomCreateEventContent,
guest_access::{GuestAccess, RoomGuestAccessEventContent},
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
join_rules::{JoinRule, RoomJoinRulesEventContent},
member::{MembershipState, RoomMemberEventContent},
name::RoomNameEventContent,
power_levels::RoomPowerLevelsEventContent,
topic::RoomTopicEventContent,
},
events::{room::message::RoomMessageEventContent, EventType},
identifiers::{EventId, RoomAliasId, RoomId, RoomName, RoomVersionId, ServerName, UserId},
};
use serde_json::value::to_raw_value;
use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard};
use tracing::warn;
pub enum AdminCommand {
RegisterAppservice(serde_yaml::Value),
ListAppservices,
SendMessage(message::MessageEventContent),
pub enum AdminRoomEvent {
ProcessMessage(String),
SendMessage(RoomMessageEventContent),
}
#[derive(Clone)]
pub struct Admin {
pub sender: mpsc::UnboundedSender<AdminCommand>,
pub sender: mpsc::UnboundedSender<AdminRoomEvent>,
}
impl Admin {
pub fn start_handler(
&self,
db: Arc<RwLock<Database>>,
mut receiver: mpsc::UnboundedReceiver<AdminCommand>,
mut receiver: mpsc::UnboundedReceiver<AdminRoomEvent>,
) {
tokio::spawn(async move {
// TODO: Use futures when we have long admin commands
@ -35,30 +51,23 @@ impl Admin {
let guard = db.read().await;
let conduit_user =
UserId::try_from(format!("@conduit:{}", guard.globals.server_name()))
.expect("@conduit:server_name is valid");
let conduit_user = UserId::parse(format!("@conduit:{}", guard.globals.server_name()))
.expect("@conduit:server_name is valid");
let conduit_room = guard
.rooms
.id_from_alias(
&format!("#admins:{}", guard.globals.server_name())
format!("#admins:{}", guard.globals.server_name())
.as_str()
.try_into()
.expect("#admins:server_name is a valid room alias"),
)
.unwrap();
let conduit_room = match conduit_room {
None => {
warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this.");
return;
}
Some(r) => r,
};
.expect("Database data for admin room alias must be valid")
.expect("Admin room must exist");
drop(guard);
let send_message = |message: message::MessageEventContent,
let send_message = |message: RoomMessageEventContent,
guard: RwLockReadGuard<'_, Database>,
mutex_lock: &MutexGuard<'_, ()>| {
guard
@ -66,7 +75,7 @@ impl Admin {
.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMessage,
content: serde_json::to_value(message)
content: to_raw_value(&message)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
@ -95,24 +104,13 @@ impl Admin {
let state_lock = mutex_state.lock().await;
match event {
AdminCommand::RegisterAppservice(yaml) => {
guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error
AdminRoomEvent::SendMessage(content) => {
send_message(content, guard, &state_lock);
}
AdminCommand::ListAppservices => {
if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::<Vec<_>>()) {
let count = appservices.len();
let output = format!(
"Appservices ({}): {}",
count,
appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
);
send_message(message::MessageEventContent::text_plain(output), guard, &state_lock);
} else {
send_message(message::MessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock);
}
}
AdminCommand::SendMessage(message) => {
send_message(message, guard, &state_lock);
AdminRoomEvent::ProcessMessage(room_message) => {
let reply_message = process_admin_message(&*guard, room_message);
send_message(reply_message, guard, &state_lock);
}
}
@ -123,7 +121,713 @@ impl Admin {
});
}
pub fn send(&self, command: AdminCommand) {
self.sender.unbounded_send(command).unwrap();
pub fn process_message(&self, room_message: String) {
self.sender
.unbounded_send(AdminRoomEvent::ProcessMessage(room_message))
.unwrap();
}
pub fn send_message(&self, message_content: RoomMessageEventContent) {
self.sender
.unbounded_send(AdminRoomEvent::SendMessage(message_content))
.unwrap();
}
}
// Parse and process a message from the admin room
fn process_admin_message(db: &Database, room_message: String) -> RoomMessageEventContent {
let mut lines = room_message.lines();
let command_line = lines.next().expect("each string has at least one line");
let body: Vec<_> = lines.collect();
let admin_command = match parse_admin_command(&command_line) {
Ok(command) => command,
Err(error) => {
let server_name = db.globals.server_name();
let message = error
.to_string()
.replace("server.name", server_name.as_str());
let html_message = usage_to_html(&message, server_name);
return RoomMessageEventContent::text_html(message, html_message);
}
};
match process_admin_command(db, admin_command, body) {
Ok(reply_message) => reply_message,
Err(error) => {
let markdown_message = format!(
"Encountered an error while handling the command:\n\
```\n{}\n```",
error,
);
let html_message = format!(
"Encountered an error while handling the command:\n\
<pre>\n{}\n</pre>",
error,
);
RoomMessageEventContent::text_html(markdown_message, html_message)
}
}
}
// Parse chat messages from the admin room into an AdminCommand object
fn parse_admin_command(command_line: &str) -> std::result::Result<AdminCommand, String> {
// Note: argv[0] is `@conduit:servername:`, which is treated as the main command
let mut argv: Vec<_> = command_line.split_whitespace().collect();
// Replace `help command` with `command --help`
// Clap has a help subcommand, but it omits the long help description.
if argv.len() > 1 && argv[1] == "help" {
argv.remove(1);
argv.push("--help");
}
// Backwards compatibility with `register_appservice`-style commands
let command_with_dashes;
if argv.len() > 1 && argv[1].contains("_") {
command_with_dashes = argv[1].replace("_", "-");
argv[1] = &command_with_dashes;
}
AdminCommand::try_parse_from(argv).map_err(|error| error.to_string())
}
#[derive(Parser)]
#[clap(name = "@conduit:server.name:", version = env!("CARGO_PKG_VERSION"))]
enum AdminCommand {
#[clap(verbatim_doc_comment)]
/// Register an appservice using its registration YAML
///
/// This command needs a YAML generated by an appservice (such as a bridge),
/// which must be provided in a Markdown code-block below the command.
///
/// Registering a new bridge using the ID of an existing bridge will replace
/// the old one.
///
/// [commandbody]
/// # ```
/// # yaml content here
/// # ```
RegisterAppservice,
/// Unregister an appservice using its ID
///
/// You can find the ID using the `list-appservices` command.
UnregisterAppservice {
/// The appservice to unregister
appservice_identifier: String,
},
/// List all the currently registered appservices
ListAppservices,
/// List users in the database
ListLocalUsers,
/// Get the auth_chain of a PDU
GetAuthChain {
/// An event ID (the $ character followed by the base64 reference hash)
event_id: Box<EventId>,
},
#[clap(verbatim_doc_comment)]
/// Parse and print a PDU from a JSON
///
/// The PDU event is only checked for validity and is not added to the
/// database.
///
/// [commandbody]
/// # ```
/// # PDU json content here
/// # ```
ParsePdu,
/// Retrieve and print a PDU by ID from the Conduit database
GetPdu {
/// An event ID (a $ followed by the base64 reference hash)
event_id: Box<EventId>,
},
/// Print database memory usage statistics
DatabaseMemoryUsage,
}
fn process_admin_command(
db: &Database,
command: AdminCommand,
body: Vec<&str>,
) -> Result<RoomMessageEventContent> {
let reply_message_content = match command {
AdminCommand::RegisterAppservice => {
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" {
let appservice_config = body[1..body.len() - 1].join("\n");
let parsed_config = serde_yaml::from_str::<serde_yaml::Value>(&appservice_config);
match parsed_config {
Ok(yaml) => match db.appservice.register_appservice(yaml) {
Ok(id) => RoomMessageEventContent::text_plain(format!(
"Appservice registered with ID: {}.",
id
)),
Err(e) => RoomMessageEventContent::text_plain(format!(
"Failed to register appservice: {}",
e
)),
},
Err(e) => RoomMessageEventContent::text_plain(format!(
"Could not parse appservice config: {}",
e
)),
}
} else {
RoomMessageEventContent::text_plain(
"Expected code block in command body. Add --help for details.",
)
}
}
AdminCommand::UnregisterAppservice {
appservice_identifier,
} => match db.appservice.unregister_appservice(&appservice_identifier) {
Ok(()) => RoomMessageEventContent::text_plain("Appservice unregistered."),
Err(e) => RoomMessageEventContent::text_plain(format!(
"Failed to unregister appservice: {}",
e
)),
},
AdminCommand::ListAppservices => {
if let Ok(appservices) = db.appservice.iter_ids().map(|ids| ids.collect::<Vec<_>>()) {
let count = appservices.len();
let output = format!(
"Appservices ({}): {}",
count,
appservices
.into_iter()
.filter_map(|r| r.ok())
.collect::<Vec<_>>()
.join(", ")
);
RoomMessageEventContent::text_plain(output)
} else {
RoomMessageEventContent::text_plain("Failed to get appservices.")
}
}
AdminCommand::ListLocalUsers => match db.users.list_local_users() {
Ok(users) => {
let mut msg: String = format!("Found {} local user account(s):\n", users.len());
msg += &users.join("\n");
RoomMessageEventContent::text_plain(&msg)
}
Err(e) => RoomMessageEventContent::text_plain(e.to_string()),
},
AdminCommand::GetAuthChain { event_id } => {
let event_id = Arc::<EventId>::from(event_id);
if let Some(event) = db.rooms.get_pdu_json(&event_id)? {
let room_id_str = event
.get("room_id")
.and_then(|val| val.as_str())
.ok_or_else(|| Error::bad_database("Invalid event in database"))?;
let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| {
Error::bad_database("Invalid room id field in event in database")
})?;
let start = Instant::now();
let count = server_server::get_auth_chain(room_id, vec![event_id], db)?.count();
let elapsed = start.elapsed();
RoomMessageEventContent::text_plain(format!(
"Loaded auth chain with length {} in {:?}",
count, elapsed
))
} else {
RoomMessageEventContent::text_plain("Event not found.")
}
}
AdminCommand::ParsePdu => {
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" {
let string = body[1..body.len() - 1].join("\n");
match serde_json::from_str(&string) {
Ok(value) => {
let event_id = EventId::parse(format!(
"${}",
// Anything higher than version3 behaves the same
ruma::signatures::reference_hash(&value, &RoomVersionId::V6)
.expect("ruma can calculate reference hashes")
))
.expect("ruma's reference hashes are valid event ids");
match serde_json::from_value::<PduEvent>(
serde_json::to_value(value).expect("value is json"),
) {
Ok(pdu) => RoomMessageEventContent::text_plain(format!(
"EventId: {:?}\n{:#?}",
event_id, pdu
)),
Err(e) => RoomMessageEventContent::text_plain(format!(
"EventId: {:?}\nCould not parse event: {}",
event_id, e
)),
}
}
Err(e) => RoomMessageEventContent::text_plain(format!(
"Invalid json in command body: {}",
e
)),
}
} else {
RoomMessageEventContent::text_plain("Expected code block in command body.")
}
}
AdminCommand::GetPdu { event_id } => {
let mut outlier = false;
let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?;
if pdu_json.is_none() {
outlier = true;
pdu_json = db.rooms.get_pdu_json(&event_id)?;
}
match pdu_json {
Some(json) => {
let json_text =
serde_json::to_string_pretty(&json).expect("canonical json is valid json");
RoomMessageEventContent::text_html(
format!(
"{}\n```json\n{}\n```",
if outlier {
"PDU is outlier"
} else {
"PDU was accepted"
},
json_text
),
format!(
"<p>{}</p>\n<pre><code class=\"language-json\">{}\n</code></pre>\n",
if outlier {
"PDU is outlier"
} else {
"PDU was accepted"
},
RawStr::new(&json_text).html_escape()
),
)
}
None => RoomMessageEventContent::text_plain("PDU not found."),
}
}
AdminCommand::DatabaseMemoryUsage => match db._db.memory_usage() {
Ok(response) => RoomMessageEventContent::text_plain(response),
Err(e) => RoomMessageEventContent::text_plain(format!(
"Failed to get database memory usage: {}",
e
)),
},
};
Ok(reply_message_content)
}
// Utility to turn clap's `--help` text to HTML.
fn usage_to_html(text: &str, server_name: &ServerName) -> String {
// Replace `@conduit:servername:-subcmdname` with `@conduit:servername: subcmdname`
let text = text.replace(
&format!("@conduit:{}:-", server_name),
&format!("@conduit:{}: ", server_name),
);
// For the conduit admin room, subcommands become main commands
let text = text.replace("SUBCOMMAND", "COMMAND");
let text = text.replace("subcommand", "command");
// Escape option names (e.g. `<element-id>`) since they look like HTML tags
let text = text.replace("<", "&lt;").replace(">", "&gt;");
// Italicize the first line (command name and version text)
let re = Regex::new("^(.*?)\n").expect("Regex compilation should not fail");
let text = re.replace_all(&text, "<em>$1</em>\n");
// Unmerge wrapped lines
let text = text.replace("\n ", " ");
// Wrap option names in backticks. The lines look like:
// -V, --version Prints version information
// And are converted to:
// <code>-V, --version</code>: Prints version information
// (?m) enables multi-line mode for ^ and $
let re = Regex::new("(?m)^ (([a-zA-Z_&;-]+(, )?)+) +(.*)$")
.expect("Regex compilation should not fail");
let text = re.replace_all(&text, "<code>$1</code>: $4");
// Look for a `[commandbody]` tag. If it exists, use all lines below it that
// start with a `#` in the USAGE section.
let mut text_lines: Vec<&str> = text.lines().collect();
let mut command_body = String::new();
if let Some(line_index) = text_lines.iter().position(|line| *line == "[commandbody]") {
text_lines.remove(line_index);
while text_lines
.get(line_index)
.map(|line| line.starts_with("#"))
.unwrap_or(false)
{
command_body += if text_lines[line_index].starts_with("# ") {
&text_lines[line_index][2..]
} else {
&text_lines[line_index][1..]
};
command_body += "[nobr]\n";
text_lines.remove(line_index);
}
}
let text = text_lines.join("\n");
// Improve the usage section
let text = if command_body.is_empty() {
// Wrap the usage line in code tags
let re = Regex::new("(?m)^USAGE:\n (@conduit:.*)$")
.expect("Regex compilation should not fail");
re.replace_all(&text, "USAGE:\n<code>$1</code>").to_string()
} else {
// Wrap the usage line in a code block, and add a yaml block example
// This makes the usage of e.g. `register-appservice` more accurate
let re =
Regex::new("(?m)^USAGE:\n (.*?)\n\n").expect("Regex compilation should not fail");
re.replace_all(&text, "USAGE:\n<pre>$1[nobr]\n[commandbodyblock]</pre>")
.replace("[commandbodyblock]", &command_body)
};
// Add HTML line-breaks
let text = text
.replace("\n\n\n", "\n\n")
.replace("\n", "<br>\n")
.replace("[nobr]<br>", "");
text
}
/// Create the admin room.
///
/// Users in this room are considered admins by conduit, and the room can be
/// used to issue admin commands by talking to the server user inside it.
pub(crate) async fn create_admin_room(db: &Database) -> Result<()> {
let room_id = RoomId::new(db.globals.server_name());
db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
let mutex_state = Arc::clone(
db.globals
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.or_default(),
);
let state_lock = mutex_state.lock().await;
// Create a user for the server
let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
.expect("@conduit:server_name is valid");
db.users.create(&conduit_user, None)?;
let mut content = RoomCreateEventContent::new(conduit_user.clone());
content.federate = true;
content.predecessor = None;
content.room_version = RoomVersionId::V6;
// 1. The room create event
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCreate,
content: to_raw_value(&content).expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 2. Make conduit bot join
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Join,
displayname: None,
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(conduit_user.to_string()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 3. Power levels
let mut users = BTreeMap::new();
users.insert(conduit_user.clone(), 100.into());
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomPowerLevels,
content: to_raw_value(&RoomPowerLevelsEventContent {
users,
..Default::default()
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.1 Join Rules
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomJoinRules,
content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.2 History Visibility
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomHistoryVisibility,
content: to_raw_value(&RoomHistoryVisibilityEventContent::new(
HistoryVisibility::Shared,
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 4.3 Guest Access
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomGuestAccess,
content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 5. Events implied by name and topic
let room_name = RoomName::parse(format!("{} Admin Room", db.globals.server_name()))
.expect("Room name is valid");
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomName,
content: to_raw_value(&RoomNameEventContent::new(Some(room_name)))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomTopic,
content: to_raw_value(&RoomTopicEventContent {
topic: format!("Manage {}", db.globals.server_name()),
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// 6. Room alias
let alias: Box<RoomAliasId> = format!("#admins:{}", db.globals.server_name())
.try_into()
.expect("#admins:server_name is a valid alias name");
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomCanonicalAlias,
content: to_raw_value(&RoomCanonicalAliasEventContent {
alias: Some(alias.clone()),
alt_aliases: Vec::new(),
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
Ok(())
}
/// Invite the user to the conduit admin room.
///
/// In conduit, this is equivalent to granting admin privileges.
pub(crate) async fn make_user_admin(
db: &Database,
user_id: &UserId,
displayname: String,
) -> Result<()> {
let admin_room_alias: Box<RoomAliasId> = format!("#admins:{}", db.globals.server_name())
.try_into()
.expect("#admins:server_name is a valid alias name");
let room_id = db
.rooms
.id_from_alias(&admin_room_alias)?
.expect("Admin room must exist");
let mutex_state = Arc::clone(
db.globals
.roomid_mutex_state
.write()
.unwrap()
.entry(room_id.clone())
.or_default(),
);
let state_lock = mutex_state.lock().await;
// Use the server user to grant the new admin's power level
let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
.expect("@conduit:server_name is valid");
// Invite and join the real user
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Invite,
displayname: None,
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(user_id.to_string()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMember,
content: to_raw_value(&RoomMemberEventContent {
membership: MembershipState::Join,
displayname: Some(displayname),
avatar_url: None,
is_direct: None,
third_party_invite: None,
blurhash: None,
reason: None,
join_authorized_via_users_server: None,
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some(user_id.to_string()),
redacts: None,
},
&user_id,
&room_id,
&db,
&state_lock,
)?;
// Set power level
let mut users = BTreeMap::new();
users.insert(conduit_user.to_owned(), 100.into());
users.insert(user_id.to_owned(), 100.into());
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomPowerLevels,
content: to_raw_value(&RoomPowerLevelsEventContent {
users,
..Default::default()
})
.expect("event is valid, we just created it"),
unsigned: None,
state_key: Some("".to_owned()),
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
// Send welcome message
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMessage,
content: to_raw_value(&RoomMessageEventContent::text_html(
"## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(),
"<h2>Thank you for trying out Conduit!</h2>\n<p>Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.</p>\n<p>Helpful links:</p>\n<blockquote>\n<p>Website: https://conduit.rs<br>Git and Documentation: https://gitlab.com/famedly/conduit<br>Report issues: https://gitlab.com/famedly/conduit/-/issues</p>\n</blockquote>\n<p>Here are some rooms you can join (by typing the command):</p>\n<p>Conduit room (Ask questions and get notified on updates):<br><code>/join #conduit:fachschaften.org</code></p>\n<p>Conduit lounge (Off-topic, only Conduit users are allowed to join)<br><code>/join #conduit-lounge:conduit.rs</code></p>\n".to_owned(),
))
.expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
redacts: None,
},
&conduit_user,
&room_id,
&db,
&state_lock,
)?;
Ok(())
}

21
src/database/appservice.rs
View file

@ -12,7 +12,9 @@ pub struct Appservice {
}
impl Appservice {
pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
/// Registers an appservice and returns the ID to the caller
///
pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<String> {
// TODO: Rumaify
let id = yaml.get("id").unwrap().as_str().unwrap();
self.id_appserviceregistrations.insert(
@ -22,8 +24,23 @@ impl Appservice {
self.cached_registrations
.write()
.unwrap()
.insert(id.to_owned(), yaml);
.insert(id.to_owned(), yaml.to_owned());
Ok(id.to_owned())
}
/// Remove an appservice registration
///
/// # Arguments
///
/// * `service_name` - the name you send to register the service previously
pub fn unregister_appservice(&self, service_name: &str) -> Result<()> {
self.id_appserviceregistrations
.remove(service_name.as_bytes())?;
self.cached_registrations
.write()
.unwrap()
.remove(service_name);
Ok(())
}

95
src/database/globals.rs
View file

@ -10,7 +10,7 @@ use std::{
collections::{BTreeMap, HashMap},
fs,
future::Future,
net::IpAddr,
net::{IpAddr, SocketAddr},
path::PathBuf,
sync::{Arc, Mutex, RwLock},
time::{Duration, Instant},
@ -39,14 +39,16 @@ pub struct Globals {
keypair: Arc<ruma::signatures::Ed25519KeyPair>,
dns_resolver: TokioAsyncResolver,
jwt_decoding_key: Option<jsonwebtoken::DecodingKey<'static>>,
federation_client: reqwest::Client,
default_client: reqwest::Client,
pub(super) server_signingkeys: Arc<dyn Tree>,
pub bad_event_ratelimiter: Arc<RwLock<HashMap<EventId, RateLimitState>>>,
pub bad_event_ratelimiter: Arc<RwLock<HashMap<Box<EventId>, RateLimitState>>>,
pub bad_signature_ratelimiter: Arc<RwLock<HashMap<Vec<String>, RateLimitState>>>,
pub servername_ratelimiter: Arc<RwLock<HashMap<Box<ServerName>, Arc<Semaphore>>>>,
pub sync_receivers: RwLock<HashMap<(UserId, Box<DeviceId>), SyncHandle>>,
pub roomid_mutex_insert: RwLock<HashMap<RoomId, Arc<Mutex<()>>>>,
pub roomid_mutex_state: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>,
pub roomid_mutex_federation: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>, // this lock will be held longer
pub sync_receivers: RwLock<HashMap<(Box<UserId>, Box<DeviceId>), SyncHandle>>,
pub roomid_mutex_insert: RwLock<HashMap<Box<RoomId>, Arc<Mutex<()>>>>,
pub roomid_mutex_state: RwLock<HashMap<Box<RoomId>, Arc<TokioMutex<()>>>>,
pub roomid_mutex_federation: RwLock<HashMap<Box<RoomId>, Arc<TokioMutex<()>>>>, // this lock will be held longer
pub rotate: RotationHandler,
}
@ -57,8 +59,7 @@ pub struct RotationHandler(broadcast::Sender<()>, broadcast::Receiver<()>);
impl RotationHandler {
pub fn new() -> Self {
let (s, r) = broadcast::channel::<()>(1);
let (s, r) = broadcast::channel(1);
Self(s, r)
}
@ -113,7 +114,7 @@ impl Globals {
.map(|key| (version, key))
})
.and_then(|(version, key)| {
ruma::signatures::Ed25519KeyPair::from_der(&key, version)
ruma::signatures::Ed25519KeyPair::from_der(key, version)
.map_err(|_| Error::bad_database("Private or public keys are invalid."))
});
@ -133,6 +134,17 @@ impl Globals {
.as_ref()
.map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static());
let default_client = reqwest_client_builder(&config)?.build()?;
let name_override = Arc::clone(&tls_name_override);
let federation_client = reqwest_client_builder(&config)?
.resolve_fn(move |domain| {
let read_guard = name_override.read().unwrap();
let (override_name, port) = read_guard.get(&domain)?;
let first_name = override_name.get(0)?;
Some(SocketAddr::new(*first_name, *port))
})
.build()?;
let s = Self {
globals,
config,
@ -142,6 +154,8 @@ impl Globals {
})?,
actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())),
tls_name_override,
federation_client,
default_client,
server_signingkeys,
jwt_decoding_key,
bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
@ -164,17 +178,16 @@ impl Globals {
&self.keypair
}
/// Returns a reqwest client which can be used to send requests.
pub fn reqwest_client(&self) -> Result<reqwest::ClientBuilder> {
let mut reqwest_client_builder = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(60 * 3))
.pool_max_idle_per_host(1);
if let Some(proxy) = self.config.proxy.to_proxy()? {
reqwest_client_builder = reqwest_client_builder.proxy(proxy);
}
/// Returns a reqwest client which can be used to send requests
pub fn default_client(&self) -> reqwest::Client {
// Client is cheap to clone (Arc wrapper) and avoids lifetime issues
self.default_client.clone()
}
Ok(reqwest_client_builder)
/// Returns a client used for resolving .well-knowns
pub fn federation_client(&self) -> reqwest::Client {
// Client is cheap to clone (Arc wrapper) and avoids lifetime issues
self.federation_client.clone()
}
#[tracing::instrument(skip(self))]
@ -211,6 +224,10 @@ impl Globals {
self.config.allow_federation
}
pub fn allow_room_creation(&self) -> bool {
self.config.allow_room_creation
}
pub fn trusted_servers(&self) -> &[Box<ServerName>] {
&self.config.trusted_servers
}
@ -223,6 +240,26 @@ impl Globals {
self.jwt_decoding_key.as_ref()
}
pub fn turn_password(&self) -> &String {
&self.config.turn_password
}
pub fn turn_ttl(&self) -> u64 {
self.config.turn_ttl
}
pub fn turn_uris(&self) -> &[String] {
&self.config.turn_uris
}
pub fn turn_username(&self) -> &String {
&self.config.turn_username
}
pub fn turn_secret(&self) -> &String {
&self.config.turn_secret
}
/// TODO: the key valid until timestamp is only honored in room version > 4
/// Remove the outdated keys and insert the new ones.
///
@ -231,7 +268,7 @@ impl Globals {
&self,
origin: &ServerName,
new_keys: ServerSigningKeys,
) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
) -> Result<BTreeMap<Box<ServerSigningKeyId>, VerifyKey>> {
// Not atomic, but this is not critical
let signingkeys = self.server_signingkeys.get(origin.as_bytes())?;
@ -270,12 +307,12 @@ impl Globals {
pub fn signing_keys_for(
&self,
origin: &ServerName,
) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
) -> Result<BTreeMap<Box<ServerSigningKeyId>, VerifyKey>> {
let signingkeys = self
.server_signingkeys
.get(origin.as_bytes())?
.and_then(|bytes| serde_json::from_slice::<ServerSigningKeys>(&bytes).ok())
.map(|keys| {
.and_then(|bytes| serde_json::from_slice(&bytes).ok())
.map(|keys: ServerSigningKeys| {
let mut tree = keys.verify_keys;
tree.extend(
keys.old_verify_keys
@ -317,3 +354,15 @@ impl Globals {
r
}
}
fn reqwest_client_builder(config: &Config) -> Result<reqwest::ClientBuilder> {
let mut reqwest_client_builder = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(60 * 3));
if let Some(proxy) = config.proxy.to_proxy()? {
reqwest_client_builder = reqwest_client_builder.proxy(proxy);
}
Ok(reqwest_client_builder)
}

83
src/database/key_backups.rs
View file

@ -4,9 +4,10 @@ use ruma::{
error::ErrorKind,
r0::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup},
},
serde::Raw,
RoomId, UserId,
};
use std::{collections::BTreeMap, convert::TryFrom, sync::Arc};
use std::{collections::BTreeMap, sync::Arc};
use super::abstraction::Tree;
@ -20,14 +21,14 @@ impl KeyBackups {
pub fn create_backup(
&self,
user_id: &UserId,
backup_metadata: &BackupAlgorithm,
backup_metadata: &Raw<BackupAlgorithm>,
globals: &super::globals::Globals,
) -> Result<String> {
let version = globals.next_count()?.to_string();
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
self.backupid_algorithm.insert(
&key,
@ -41,7 +42,7 @@ impl KeyBackups {
pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
self.backupid_algorithm.remove(&key)?;
self.backupid_etag.remove(&key)?;
@ -59,12 +60,12 @@ impl KeyBackups {
&self,
user_id: &UserId,
version: &str,
backup_metadata: &BackupAlgorithm,
backup_metadata: &Raw<BackupAlgorithm>,
globals: &super::globals::Globals,
) -> Result<String> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
if self.backupid_algorithm.get(&key)?.is_none() {
return Err(Error::BadRequest(
@ -73,15 +74,11 @@ impl KeyBackups {
));
}
self.backupid_algorithm.insert(
&key,
&serde_json::to_string(backup_metadata)
.expect("BackupAlgorithm::to_string always works")
.as_bytes(),
)?;
self.backupid_algorithm
.insert(&key, backup_metadata.json().get().as_bytes())?;
self.backupid_etag
.insert(&key, &globals.next_count()?.to_be_bytes())?;
Ok(version.to_string())
Ok(version.to_owned())
}
pub fn get_latest_backup_version(&self, user_id: &UserId) -> Result<Option<String>> {
@ -94,18 +91,21 @@ impl KeyBackups {
.iter_from(&last_possible_key, true)
.take_while(move |(k, _)| k.starts_with(&prefix))
.next()
.map_or(Ok(None), |(key, _)| {
.map(|(key, _)| {
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))
.map(Some)
})
.transpose()
}
pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
pub fn get_latest_backup(
&self,
user_id: &UserId,
) -> Result<Option<(String, Raw<BackupAlgorithm>)>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
let mut last_possible_key = prefix.clone();
@ -115,7 +115,7 @@ impl KeyBackups {
.iter_from(&last_possible_key, true)
.take_while(move |(k, _)| k.starts_with(&prefix))
.next()
.map_or(Ok(None), |(key, value)| {
.map(|(key, value)| {
let version = utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
@ -123,16 +123,21 @@ impl KeyBackups {
)
.map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?;
Ok(Some((
Ok((
version,
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("Algorithm in backupid_algorithm is invalid.")
})?,
)))
))
})
.transpose()
}
pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<BackupAlgorithm>> {
pub fn get_backup(
&self,
user_id: &UserId,
version: &str,
) -> Result<Option<Raw<BackupAlgorithm>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(version.as_bytes());
@ -151,7 +156,7 @@ impl KeyBackups {
version: &str,
room_id: &RoomId,
session_id: &str,
key_data: &KeyBackupData,
key_data: &Raw<KeyBackupData>,
globals: &super::globals::Globals,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
@ -173,10 +178,8 @@ impl KeyBackups {
key.push(0xff);
key.extend_from_slice(session_id.as_bytes());
self.backupkeyid_backup.insert(
&key,
&serde_json::to_vec(&key_data).expect("KeyBackupData::to_vec always works"),
)?;
self.backupkeyid_backup
.insert(&key, key_data.json().get().as_bytes())?;
Ok(())
}
@ -192,7 +195,7 @@ impl KeyBackups {
pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result<String> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
Ok(utils::u64_from_bytes(
&self
@ -208,13 +211,13 @@ impl KeyBackups {
&self,
user_id: &UserId,
version: &str,
) -> Result<BTreeMap<RoomId, RoomKeyBackup>> {
) -> Result<BTreeMap<Box<RoomId>, RoomKeyBackup>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(version.as_bytes());
prefix.push(0xff);
let mut rooms = BTreeMap::<RoomId, RoomKeyBackup>::new();
let mut rooms = BTreeMap::<Box<RoomId>, RoomKeyBackup>::new();
for result in self
.backupkeyid_backup
@ -223,15 +226,15 @@ impl KeyBackups {
let mut parts = key.rsplit(|&b| b == 0xff);
let session_id =
utils::string_from_bytes(&parts.next().ok_or_else(|| {
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| {
Error::bad_database("backupkeyid_backup session_id is invalid.")
})?;
let room_id = RoomId::try_from(
utils::string_from_bytes(&parts.next().ok_or_else(|| {
let room_id = RoomId::parse(
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
@ -265,7 +268,7 @@ impl KeyBackups {
user_id: &UserId,
version: &str,
room_id: &RoomId,
) -> Result<BTreeMap<String, KeyBackupData>> {
) -> Result<BTreeMap<String, Raw<KeyBackupData>>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(version.as_bytes());
@ -280,7 +283,7 @@ impl KeyBackups {
let mut parts = key.rsplit(|&b| b == 0xff);
let session_id =
utils::string_from_bytes(&parts.next().ok_or_else(|| {
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| {
@ -303,7 +306,7 @@ impl KeyBackups {
version: &str,
room_id: &RoomId,
session_id: &str,
) -> Result<Option<KeyBackupData>> {
) -> Result<Option<Raw<KeyBackupData>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(version.as_bytes());
@ -325,7 +328,7 @@ impl KeyBackups {
pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
key.push(0xff);
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
@ -343,9 +346,9 @@ impl KeyBackups {
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.extend_from_slice(&room_id.as_bytes());
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
@ -364,11 +367,11 @@ impl KeyBackups {
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&version.as_bytes());
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.extend_from_slice(&room_id.as_bytes());
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.extend_from_slice(&session_id.as_bytes());
key.extend_from_slice(session_id.as_bytes());
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
self.backupkeyid_backup.remove(&outdated_key)?;

7
src/database/media.rs
View file

@ -4,7 +4,10 @@ use image::{imageops::FilterType, GenericImageView};
use super::abstraction::Tree;
use crate::{utils, Error, Result};
use std::{mem, sync::Arc};
use tokio::{fs::File, io::AsyncReadExt, io::AsyncWriteExt};
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncWriteExt},
};
pub struct FileMeta {
pub content_disposition: Option<String>,
@ -168,7 +171,7 @@ impl Media {
/// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
pub async fn get_thumbnail(
&self,
mxc: String,
mxc: &str,
globals: &Globals,
width: u32,
height: u32,

52
src/database/pusher.rs
View file

@ -9,15 +9,17 @@ use ruma::{
},
IncomingResponse, OutgoingRequest, SendAccessToken,
},
events::{room::power_levels::PowerLevelsEventContent, AnySyncRoomEvent, EventType},
identifiers::RoomName,
events::{
room::{name::RoomNameEventContent, power_levels::RoomPowerLevelsEventContent},
AnySyncRoomEvent, EventType,
},
push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak},
serde::Raw,
uint, RoomId, UInt, UserId,
};
use tracing::{error, info, warn};
use std::{convert::TryFrom, fmt::Debug, mem, sync::Arc};
use std::{fmt::Debug, mem, sync::Arc};
use super::abstraction::Tree;
@ -113,11 +115,7 @@ where
//*reqwest_request.timeout_mut() = Some(Duration::from_secs(5));
let url = reqwest_request.url().clone();
let response = globals
.reqwest_client()?
.build()?
.execute(reqwest_request)
.await;
let response = globals.default_client().execute(reqwest_request).await;
match response {
Ok(mut response) => {
@ -177,11 +175,11 @@ pub async fn send_push_notice(
let mut notify = None;
let mut tweaks = Vec::new();
let power_levels: PowerLevelsEventContent = db
let power_levels: RoomPowerLevelsEventContent = db
.rooms
.room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")?
.map(|ev| {
serde_json::from_value(ev.content.clone())
serde_json::from_str(ev.content.get())
.map_err(|_| Error::bad_database("invalid m.room.power_levels event"))
})
.transpose()?
@ -226,17 +224,17 @@ pub async fn send_push_notice(
pub fn get_actions<'a>(
user: &UserId,
ruleset: &'a Ruleset,
power_levels: &PowerLevelsEventContent,
power_levels: &RoomPowerLevelsEventContent,
pdu: &Raw<AnySyncRoomEvent>,
room_id: &RoomId,
db: &Database,
) -> Result<&'a [Action]> {
let ctx = PushConditionRoomCtx {
room_id: room_id.clone(),
room_id: room_id.to_owned(),
member_count: 10_u32.into(), // TODO: get member count efficiently
user_display_name: db
.users
.displayname(&user)?
.displayname(user)?
.unwrap_or_else(|| user.localpart().to_owned()),
users_power_levels: power_levels.users.clone(),
default_power_level: power_levels.users_default,
@ -275,7 +273,7 @@ async fn send_notice(
let mut data_minus_url = pusher.data.clone();
// The url must be stripped off according to spec
data_minus_url.url = None;
device.data = Some(data_minus_url);
device.data = data_minus_url;
// Tweaks are only added if the format is NOT event_id_only
if !event_id_only {
@ -302,7 +300,7 @@ async fn send_notice(
if event_id_only {
send_request(
&db.globals,
&url,
url,
send_event_notification::v1::Request::new(notifi),
)
.await?;
@ -318,21 +316,23 @@ async fn send_notice(
let user_name = db.users.displayname(&event.sender)?;
notifi.sender_display_name = user_name.as_deref();
let room_name = db
.rooms
.room_state_get(&event.room_id, &EventType::RoomName, "")?
.map(|pdu| match pdu.content.get("name") {
Some(serde_json::Value::String(s)) => {
Some(Box::<RoomName>::try_from(&**s).expect("room name is valid"))
}
_ => None,
})
.flatten();
let room_name = if let Some(room_name_pdu) =
db.rooms
.room_state_get(&event.room_id, &EventType::RoomName, "")?
{
serde_json::from_str::<RoomNameEventContent>(room_name_pdu.content.get())
.map_err(|_| Error::bad_database("Invalid room name event in database."))?
.name
} else {
None
};
notifi.room_name = room_name.as_deref();
send_request(
&db.globals,
&url,
url,
send_event_notification::v1::Request::new(notifi),
)
.await?;

992
src/database/rooms.rs
View file

File diff suppressed because it is too large Load diff

88
src/database/rooms/edus.rs
View file

@ -11,7 +11,6 @@ use ruma::{
};
use std::{
collections::{HashMap, HashSet},
convert::{TryFrom, TryInto},
mem,
sync::Arc,
};
@ -60,7 +59,7 @@ impl RoomEdus {
let mut room_latest_id = prefix;
room_latest_id.extend_from_slice(&globals.next_count()?.to_be_bytes());
room_latest_id.push(0xff);
room_latest_id.extend_from_slice(&user_id.as_bytes());
room_latest_id.extend_from_slice(user_id.as_bytes());
self.readreceiptid_readreceipt.insert(
&room_latest_id,
@ -76,8 +75,13 @@ impl RoomEdus {
&'a self,
room_id: &RoomId,
since: u64,
) -> impl Iterator<Item = Result<(UserId, u64, Raw<ruma::events::AnySyncEphemeralRoomEvent>)>> + 'a
{
) -> impl Iterator<
Item = Result<(
Box<UserId>,
u64,
Raw<ruma::events::AnySyncEphemeralRoomEvent>,
)>,
> + 'a {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
let prefix2 = prefix.clone();
@ -92,7 +96,7 @@ impl RoomEdus {
let count =
utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::<u64>()])
.map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?;
let user_id = UserId::try_from(
let user_id = UserId::parse(
utils::string_from_bytes(&k[prefix.len() + mem::size_of::<u64>() + 1..])
.map_err(|_| {
Error::bad_database("Invalid readreceiptid userid bytes in db.")
@ -126,7 +130,7 @@ impl RoomEdus {
) -> Result<()> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&user_id.as_bytes());
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_privateread
.insert(&key, &count.to_be_bytes())?;
@ -142,7 +146,7 @@ impl RoomEdus {
pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&user_id.as_bytes());
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_privateread
.get(&key)?
@ -157,16 +161,17 @@ impl RoomEdus {
pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(&user_id.as_bytes());
key.extend_from_slice(user_id.as_bytes());
Ok(self
.roomuserid_lastprivatereadupdate
.get(&key)?
.map_or(Ok::<_, Error>(None), |bytes| {
Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.")
})?))
})?
})
})
.transpose()?
.unwrap_or(0))
}
@ -193,7 +198,7 @@ impl RoomEdus {
.insert(&room_typing_id, &*user_id.as_bytes())?;
self.roomid_lasttypingupdate
.insert(&room_id.as_bytes(), &count)?;
.insert(room_id.as_bytes(), &count)?;
Ok(())
}
@ -224,7 +229,7 @@ impl RoomEdus {
if found_outdated {
self.roomid_lasttypingupdate
.insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
.insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
}
Ok(())
@ -268,7 +273,7 @@ impl RoomEdus {
if found_outdated {
self.roomid_lasttypingupdate
.insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
.insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
}
Ok(())
@ -285,12 +290,13 @@ impl RoomEdus {
Ok(self
.roomid_lasttypingupdate
.get(&room_id.as_bytes())?
.map_or(Ok::<_, Error>(None), |bytes| {
Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
.get(room_id.as_bytes())?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
})?))
})?
})
})
.transpose()?
.unwrap_or(0))
}
@ -303,17 +309,13 @@ impl RoomEdus {
let mut user_ids = HashSet::new();
for user_id in self
.typingid_userid
.scan_prefix(prefix)
.map(|(_, user_id)| {
UserId::try_from(utils::string_from_bytes(&user_id).map_err(|_| {
Error::bad_database("User ID in typingid_userid is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))
})
{
user_ids.insert(user_id?);
for (_, user_id) in self.typingid_userid.scan_prefix(prefix) {
let user_id = UserId::parse(utils::string_from_bytes(&user_id).map_err(|_| {
Error::bad_database("User ID in typingid_userid is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))?;
user_ids.insert(user_id);
}
Ok(SyncEphemeralRoomEvent {
@ -331,7 +333,7 @@ impl RoomEdus {
&self,
user_id: &UserId,
room_id: &RoomId,
presence: ruma::events::presence::PresenceEvent,
presence: PresenceEvent,
globals: &super::super::globals::Globals,
) -> Result<()> {
// TODO: Remove old entry? Or maybe just wipe completely from time to time?
@ -342,7 +344,7 @@ impl RoomEdus {
presence_id.push(0xff);
presence_id.extend_from_slice(&count);
presence_id.push(0xff);
presence_id.extend_from_slice(&presence.sender.as_bytes());
presence_id.extend_from_slice(presence.sender.as_bytes());
self.presenceid_presence.insert(
&presence_id,
@ -361,7 +363,7 @@ impl RoomEdus {
#[tracing::instrument(skip(self))]
pub fn ping_presence(&self, user_id: &UserId) -> Result<()> {
self.userid_lastpresenceupdate.insert(
&user_id.as_bytes(),
user_id.as_bytes(),
&utils::millis_since_unix_epoch().to_be_bytes(),
)?;
@ -371,7 +373,7 @@ impl RoomEdus {
/// Returns the timestamp of the last presence update of this user in millis since the unix epoch.
pub fn last_presence_update(&self, user_id: &UserId) -> Result<Option<u64>> {
self.userid_lastpresenceupdate
.get(&user_id.as_bytes())?
.get(user_id.as_bytes())?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid timestamp in userid_lastpresenceupdate.")
@ -394,12 +396,12 @@ impl RoomEdus {
presence_id.push(0xff);
presence_id.extend_from_slice(&last_update.to_be_bytes());
presence_id.push(0xff);
presence_id.extend_from_slice(&user_id.as_bytes());
presence_id.extend_from_slice(user_id.as_bytes());
self.presenceid_presence
.get(&presence_id)?
.map(|value| {
let mut presence = serde_json::from_slice::<PresenceEvent>(&value)
let mut presence: PresenceEvent = serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("Invalid presence event in db."))?;
let current_timestamp: UInt = utils::millis_since_unix_epoch()
.try_into()
@ -447,7 +449,7 @@ impl RoomEdus {
{
// Send new presence events to set the user offline
let count = globals.next_count()?.to_be_bytes();
let user_id = utils::string_from_bytes(&user_id_bytes)
let user_id: Box<_> = utils::string_from_bytes(&user_id_bytes)
.map_err(|_| {
Error::bad_database("Invalid UserId bytes in userid_lastpresenceupdate.")
})?
@ -473,14 +475,14 @@ impl RoomEdus {
presence: PresenceState::Offline,
status_msg: None,
},
sender: user_id.clone(),
sender: user_id.to_owned(),
})
.expect("PresenceEvent can be serialized"),
)?;
}
self.userid_lastpresenceupdate.insert(
&user_id.as_bytes(),
user_id.as_bytes(),
&utils::millis_since_unix_epoch().to_be_bytes(),
)?;
}
@ -496,7 +498,7 @@ impl RoomEdus {
since: u64,
_rooms: &super::Rooms,
_globals: &super::super::globals::Globals,
) -> Result<HashMap<UserId, PresenceEvent>> {
) -> Result<HashMap<Box<UserId>, PresenceEvent>> {
//self.presence_maintain(rooms, globals)?;
let mut prefix = room_id.as_bytes().to_vec();
@ -511,7 +513,7 @@ impl RoomEdus {
.iter_from(&*first_possible_edu, false)
.take_while(|(key, _)| key.starts_with(&prefix))
{
let user_id = UserId::try_from(
let user_id = UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
@ -521,7 +523,7 @@ impl RoomEdus {
)
.map_err(|_| Error::bad_database("Invalid UserId in presenceid_presence."))?;
let mut presence = serde_json::from_slice::<PresenceEvent>(&value)
let mut presence: PresenceEvent = serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("Invalid presence event in db."))?;
let current_timestamp: UInt = utils::millis_since_unix_epoch()

145
src/database/sending.rs
View file

@ -1,6 +1,5 @@
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
fmt::Debug,
sync::Arc,
time::{Duration, Instant},
@ -27,7 +26,7 @@ use ruma::{
OutgoingRequest,
},
device_id,
events::{push_rules, AnySyncEphemeralRoomEvent, EventType},
events::{push_rules::PushRulesEvent, AnySyncEphemeralRoomEvent, EventType},
push,
receipt::ReceiptType,
uint, MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
@ -58,9 +57,9 @@ impl OutgoingKind {
}
OutgoingKind::Push(user, pushkey) => {
let mut p = b"$".to_vec();
p.extend_from_slice(&user);
p.extend_from_slice(user);
p.push(0xff);
p.extend_from_slice(&pushkey);
p.extend_from_slice(pushkey);
p
}
OutgoingKind::Normal(server) => {
@ -84,7 +83,7 @@ pub enum SendingEventType {
pub struct Sending {
/// The state for a given state hash.
pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content
pub(super) servernameevent_data: Arc<dyn Tree>, // ServernameEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content
pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / Id (for edus), Data = EDU content
pub(super) maximum_requests: Arc<Semaphore>,
pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
@ -165,13 +164,13 @@ impl Sending {
}
// Find events that have been added since starting the last request
let new_events = guard.sending.servernameevent_data
let new_events: Vec<_> = guard.sending.servernameevent_data
.scan_prefix(prefix.clone())
.filter_map(|(k, v)| {
Self::parse_servercurrentevent(&k, v).ok().map(|ev| (ev, k))
})
.take(30)
.collect::<Vec<_>>();
.collect::<>();
// TODO: find edus
@ -179,8 +178,8 @@ impl Sending {
// Insert pdus we found
for (e, key) in &new_events {
let value = if let SendingEventType::Edu(value) = &e.1 { &**value } else { &[] };
guard.sending.servercurrentevent_data.insert(&key, value).unwrap();
guard.sending.servernameevent_data.remove(&key).unwrap();
guard.sending.servercurrentevent_data.insert(key, value).unwrap();
guard.sending.servernameevent_data.remove(key).unwrap();
}
drop(guard);
@ -344,8 +343,8 @@ impl Sending {
continue;
}
let event =
serde_json::from_str::<AnySyncEphemeralRoomEvent>(&read_receipt.json().get())
let event: AnySyncEphemeralRoomEvent =
serde_json::from_str(read_receipt.json().get())
.map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?;
let federation_event = match event {
AnySyncEphemeralRoomEvent::Receipt(r) => {
@ -397,8 +396,8 @@ impl Sending {
// Because synapse resyncs, we can just insert dummy data
let edu = Edu::DeviceListUpdate(DeviceListUpdateContent {
user_id,
device_id: device_id!("dummy"),
device_display_name: "Dummy".to_owned(),
device_id: device_id!("dummy").to_owned(),
device_display_name: Some("Dummy".to_owned()),
stream_id: uint!(1),
prev_id: Vec::new(),
deleted: None,
@ -423,13 +422,23 @@ impl Sending {
Ok(())
}
#[tracing::instrument(skip(self, server, pdu_id))]
pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(pdu_id);
self.servernameevent_data.insert(&key, &[])?;
self.sender.unbounded_send((key, vec![])).unwrap();
#[tracing::instrument(skip(self, servers, pdu_id))]
pub fn send_pdu<I: Iterator<Item = Box<ServerName>>>(
&self,
servers: I,
pdu_id: &[u8],
) -> Result<()> {
let mut batch = servers.map(|server| {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(pdu_id);
self.sender.unbounded_send((key.clone(), vec![])).unwrap();
(key, Vec::new())
});
self.servernameevent_data.insert_batch(&mut batch)?;
Ok(())
}
@ -470,12 +479,32 @@ impl Sending {
hash.as_ref().to_owned()
}
/// Cleanup event data
/// Used for instance after we remove an appservice registration
///
#[tracing::instrument(skip(self))]
pub fn cleanup_events(&self, key_id: &str) -> Result<()> {
let mut prefix = b"+".to_vec();
prefix.extend_from_slice(key_id.as_bytes());
prefix.push(0xff);
for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) {
self.servercurrentevent_data.remove(&key).unwrap();
}
for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) {
self.servernameevent_data.remove(&key).unwrap();
}
Ok(())
}
#[tracing::instrument(skip(db, events, kind))]
async fn handle_events(
kind: OutgoingKind,
events: Vec<SendingEventType>,
db: Arc<RwLock<Database>>,
) -> std::result::Result<OutgoingKind, (OutgoingKind, Error)> {
) -> Result<OutgoingKind, (OutgoingKind, Error)> {
let db = db.read().await;
match &kind {
@ -486,7 +515,7 @@ impl Sending {
match event {
SendingEventType::Pdu(pdu_id) => {
pdu_jsons.push(db.rooms
.get_pdu_from_id(&pdu_id)
.get_pdu_from_id(pdu_id)
.map_err(|e| (kind.clone(), e))?
.ok_or_else(|| {
(
@ -510,11 +539,18 @@ impl Sending {
&db.globals,
db.appservice
.get_registration(server.as_str())
.unwrap()
.unwrap(), // TODO: handle error
.map_err(|e| (kind.clone(), e))?
.ok_or_else(|| {
(
kind.clone(),
Error::bad_database(
"[Appservice] Could not load registration from db.",
),
)
})?,
appservice::event::push_events::v1::Request {
events: &pdu_jsons,
txn_id: &base64::encode_config(
txn_id: (&*base64::encode_config(
Self::calculate_hash(
&events
.iter()
@ -524,7 +560,8 @@ impl Sending {
.collect::<Vec<_>>(),
),
base64::URL_SAFE_NO_PAD,
),
))
.into(),
},
)
.await
@ -543,7 +580,7 @@ impl Sending {
SendingEventType::Pdu(pdu_id) => {
pdus.push(
db.rooms
.get_pdu_from_id(&pdu_id)
.get_pdu_from_id(pdu_id)
.map_err(|e| (kind.clone(), e))?
.ok_or_else(|| {
(
@ -563,23 +600,28 @@ impl Sending {
for pdu in pdus {
// Redacted events are not notification targets (we don't send push for them)
if pdu.unsigned.get("redacted_because").is_some() {
continue;
if let Some(unsigned) = &pdu.unsigned {
if let Ok(unsigned) =
serde_json::from_str::<serde_json::Value>(unsigned.get())
{
if unsigned.get("redacted_because").is_some() {
continue;
}
}
}
let userid =
UserId::try_from(utils::string_from_bytes(user).map_err(|_| {
(
kind.clone(),
Error::bad_database("Invalid push user string in db."),
)
})?)
.map_err(|_| {
(
kind.clone(),
Error::bad_database("Invalid push user id in db."),
)
})?;
let userid = UserId::parse(utils::string_from_bytes(user).map_err(|_| {
(
kind.clone(),
Error::bad_database("Invalid push user string in db."),
)
})?)
.map_err(|_| {
(
kind.clone(),
Error::bad_database("Invalid push user id in db."),
)
})?;
let mut senderkey = user.clone();
senderkey.push(0xff);
@ -596,9 +638,9 @@ impl Sending {
let rules_for_user = db
.account_data
.get::<push_rules::PushRulesEvent>(None, &userid, EventType::PushRules)
.get(None, &userid, EventType::PushRules)
.unwrap_or_default()
.map(|ev| ev.content.global)
.map(|ev: PushRulesEvent| ev.content.global)
.unwrap_or_else(|| push::Ruleset::server_default(&userid));
let unread: UInt = db
@ -636,7 +678,7 @@ impl Sending {
// TODO: check room version and remove event_id if needed
let raw = PduEvent::convert_to_outgoing_federation_event(
db.rooms
.get_pdu_json_from_id(&pdu_id)
.get_pdu_json_from_id(pdu_id)
.map_err(|e| (OutgoingKind::Normal(server.clone()), e))?
.ok_or_else(|| {
(
@ -667,7 +709,7 @@ impl Sending {
pdus: &pdu_jsons,
edus: &edu_jsons,
origin_server_ts: MilliSecondsSinceUnixEpoch::now(),
transaction_id: &base64::encode_config(
transaction_id: (&*base64::encode_config(
Self::calculate_hash(
&events
.iter()
@ -677,7 +719,8 @@ impl Sending {
.collect::<Vec<_>>(),
),
base64::URL_SAFE_NO_PAD,
),
))
.into(),
},
)
.await
@ -711,12 +754,12 @@ impl Sending {
let event = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let server = utils::string_from_bytes(&server).map_err(|_| {
let server = utils::string_from_bytes(server).map_err(|_| {
Error::bad_database("Invalid server bytes in server_currenttransaction")
})?;
(
OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
OutgoingKind::Appservice(ServerName::parse(server).map_err(|_| {
Error::bad_database("Invalid server string in server_currenttransaction")
})?),
if value.is_empty() {
@ -750,12 +793,12 @@ impl Sending {
let event = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let server = utils::string_from_bytes(&server).map_err(|_| {
let server = utils::string_from_bytes(server).map_err(|_| {
Error::bad_database("Invalid server bytes in server_currenttransaction")
})?;
(
OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
OutgoingKind::Normal(ServerName::parse(server).map_err(|_| {
Error::bad_database("Invalid server string in server_currenttransaction")
})?),
if value.is_empty() {

6
src/database/transaction_ids.rs
View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use crate::Result;
use ruma::{DeviceId, UserId};
use ruma::{identifiers::TransactionId, DeviceId, UserId};
use super::abstraction::Tree;
@ -14,7 +14,7 @@ impl TransactionIds {
&self,
user_id: &UserId,
device_id: Option<&DeviceId>,
txn_id: &str,
txn_id: &TransactionId,
data: &[u8],
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
@ -32,7 +32,7 @@ impl TransactionIds {
&self,
user_id: &UserId,
device_id: Option<&DeviceId>,
txn_id: &str,
txn_id: &TransactionId,
) -> Result<Option<Vec<u8>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);

60
src/database/uiaa.rs
View file

@ -1,11 +1,14 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::RwLock;
use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result};
use ruma::{
api::client::{
error::ErrorKind,
r0::uiaa::{
IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, UiaaInfo,
AuthType, IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId,
UiaaInfo,
},
},
signatures::CanonicalJsonValue,
@ -17,7 +20,8 @@ use super::abstraction::Tree;
pub struct Uiaa {
pub(super) userdevicesessionid_uiaainfo: Arc<dyn Tree>, // User-interactive authentication
pub(super) userdevicesessionid_uiaarequest: Arc<dyn Tree>, // UiaaRequest = canonical json value
pub(super) userdevicesessionid_uiaarequest:
RwLock<BTreeMap<(Box<UserId>, Box<DeviceId>, String), CanonicalJsonValue>>,
}
impl Uiaa {
@ -54,7 +58,7 @@ impl Uiaa {
) -> Result<(bool, UiaaInfo)> {
let mut uiaainfo = auth
.session()
.map(|session| self.get_uiaa_session(&user_id, &device_id, session))
.map(|session| self.get_uiaa_session(user_id, device_id, session))
.unwrap_or_else(|| Ok(uiaainfo.clone()))?;
if uiaainfo.session.is_none() {
@ -99,10 +103,10 @@ impl Uiaa {
}
// Password was correct! Let's add it to `completed`
uiaainfo.completed.push("m.login.password".to_owned());
uiaainfo.completed.push(AuthType::Password);
}
IncomingAuthData::Dummy(_) => {
uiaainfo.completed.push("m.login.dummy".to_owned());
uiaainfo.completed.push(AuthType::Dummy);
}
k => error!("type not supported: {:?}", k),
}
@ -146,16 +150,13 @@ impl Uiaa {
session: &str,
request: &CanonicalJsonValue,
) -> Result<()> {
let mut userdevicesessionid = user_id.as_bytes().to_vec();
userdevicesessionid.push(0xff);
userdevicesessionid.extend_from_slice(device_id.as_bytes());
userdevicesessionid.push(0xff);
userdevicesessionid.extend_from_slice(session.as_bytes());
self.userdevicesessionid_uiaarequest.insert(
&userdevicesessionid,
&serde_json::to_vec(request).expect("json value to vec always works"),
)?;
self.userdevicesessionid_uiaarequest
.write()
.unwrap()
.insert(
(user_id.to_owned(), device_id.to_owned(), session.to_owned()),
request.to_owned(),
);
Ok(())
}
@ -165,25 +166,12 @@ impl Uiaa {
user_id: &UserId,
device_id: &DeviceId,
session: &str,
) -> Result<Option<CanonicalJsonValue>> {
let mut userdevicesessionid = user_id.as_bytes().to_vec();
userdevicesessionid.push(0xff);
userdevicesessionid.extend_from_slice(device_id.as_bytes());
userdevicesessionid.push(0xff);
userdevicesessionid.extend_from_slice(session.as_bytes());
) -> Option<CanonicalJsonValue> {
self.userdevicesessionid_uiaarequest
.get(&userdevicesessionid)?
.map_or(Ok(None), |bytes| {
Ok::<_, Error>(Some(
serde_json::from_str::<CanonicalJsonValue>(
&utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid uiaa request bytes in db.")
})?,
)
.map_err(|_| Error::bad_database("Invalid uiaa request in db."))?,
))
})
.read()
.unwrap()
.get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned()))
.map(|j| j.to_owned())
}
fn update_uiaa_session(
@ -224,7 +212,7 @@ impl Uiaa {
userdevicesessionid.push(0xff);
userdevicesessionid.extend_from_slice(session.as_bytes());
let uiaainfo = serde_json::from_slice::<UiaaInfo>(
serde_json::from_slice(
&self
.userdevicesessionid_uiaainfo
.get(&userdevicesessionid)?
@ -233,8 +221,6 @@ impl Uiaa {
"UIAA session does not exist.",
))?,
)
.map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))?;
Ok(uiaainfo)
.map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))
}
}

280
src/database/users.rs
View file

@ -1,13 +1,17 @@
use crate::{utils, Error, Result};
use ruma::{
api::client::{error::ErrorKind, r0::device::Device},
api::client::{
error::ErrorKind,
r0::{device::Device, filter::IncomingFilterDefinition},
},
encryption::{CrossSigningKey, DeviceKeys, OneTimeKey},
events::{AnyToDeviceEvent, EventType},
identifiers::MxcUri,
serde::Raw,
DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, UInt, UserId,
DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt,
UserId,
};
use std::{collections::BTreeMap, convert::TryFrom, mem, sync::Arc};
use std::{collections::BTreeMap, mem, sync::Arc};
use tracing::warn;
use super::abstraction::Tree;
@ -30,6 +34,8 @@ pub struct Users {
pub(super) userid_selfsigningkeyid: Arc<dyn Tree>,
pub(super) userid_usersigningkeyid: Arc<dyn Tree>,
pub(super) userfilterid_filter: Arc<dyn Tree>, // UserFilterId = UserId + FilterId
pub(super) todeviceid_events: Arc<dyn Tree>, // ToDeviceId = UserId + DeviceId + Count
}
@ -53,6 +59,21 @@ impl Users {
.is_empty())
}
/// Check if a user is an admin
#[tracing::instrument(skip(self, user_id, rooms, globals))]
pub fn is_admin(
&self,
user_id: &UserId,
rooms: &super::rooms::Rooms,
globals: &super::globals::Globals,
) -> Result<bool> {
let admin_room_alias_id = RoomAliasId::parse(format!("#admins:{}", globals.server_name()))
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;
let admin_room_id = rooms.id_from_alias(&admin_room_alias_id)?.unwrap();
rooms.is_joined(user_id, &admin_room_id)
}
/// Create a new user account on this homeserver.
#[tracing::instrument(skip(self, user_id, password))]
pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> {
@ -68,7 +89,7 @@ impl Users {
/// Find out which user an access token belongs to.
#[tracing::instrument(skip(self, token))]
pub fn find_from_token(&self, token: &str) -> Result<Option<(UserId, String)>> {
pub fn find_from_token(&self, token: &str) -> Result<Option<(Box<UserId>, String)>> {
self.token_userdeviceid
.get(token.as_bytes())?
.map_or(Ok(None), |bytes| {
@ -81,13 +102,13 @@ impl Users {
})?;
Ok(Some((
UserId::try_from(utils::string_from_bytes(&user_bytes).map_err(|_| {
UserId::parse(utils::string_from_bytes(user_bytes).map_err(|_| {
Error::bad_database("User ID in token_userdeviceid is invalid unicode.")
})?)
.map_err(|_| {
Error::bad_database("User ID in token_userdeviceid is invalid.")
})?,
utils::string_from_bytes(&device_bytes).map_err(|_| {
utils::string_from_bytes(device_bytes).map_err(|_| {
Error::bad_database("Device ID in token_userdeviceid is invalid.")
})?,
)))
@ -96,15 +117,51 @@ impl Users {
/// Returns an iterator over all users on this homeserver.
#[tracing::instrument(skip(self))]
pub fn iter(&self) -> impl Iterator<Item = Result<UserId>> + '_ {
pub fn iter(&self) -> impl Iterator<Item = Result<Box<UserId>>> + '_ {
self.userid_password.iter().map(|(bytes, _)| {
UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("User ID in userid_password is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("User ID in userid_password is invalid."))
})
}
/// Returns a list of local users as list of usernames.
///
/// A user account is considered `local` if the length of it's password is greater then zero.
#[tracing::instrument(skip(self))]
pub fn list_local_users(&self) -> Result<Vec<String>> {
let users: Vec<String> = self
.userid_password
.iter()
.filter_map(|(username, pw)| self.get_username_with_valid_password(&username, &pw))
.collect();
Ok(users)
}
/// Will only return with Some(username) if the password was not empty and the
/// username could be successfully parsed.
/// If utils::string_from_bytes(...) returns an error that username will be skipped
/// and the error will be logged.
#[tracing::instrument(skip(self))]
fn get_username_with_valid_password(&self, username: &[u8], password: &[u8]) -> Option<String> {
// A valid password is not empty
if password.is_empty() {
None
} else {
match utils::string_from_bytes(username) {
Ok(u) => Some(u),
Err(e) => {
warn!(
"Failed to parse username while calling get_local_users(): {}",
e.to_string()
);
None
}
}
}
}
/// Returns the password hash for the given user.
#[tracing::instrument(skip(self, user_id))]
pub fn password_hash(&self, user_id: &UserId) -> Result<Option<String>> {
@ -121,7 +178,7 @@ impl Users {
#[tracing::instrument(skip(self, user_id, password))]
pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> {
if let Some(password) = password {
if let Ok(hash) = utils::calculate_hash(&password) {
if let Ok(hash) = utils::calculate_hash(password) {
self.userid_password
.insert(user_id.as_bytes(), hash.as_bytes())?;
Ok(())
@ -164,20 +221,21 @@ impl Users {
/// Get the avatar_url of a user.
#[tracing::instrument(skip(self, user_id))]
pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<MxcUri>> {
pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<Box<MxcUri>>> {
self.userid_avatarurl
.get(user_id.as_bytes())?
.map(|bytes| {
let s = utils::string_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?;
MxcUri::try_from(s).map_err(|_| Error::bad_database("Avatar URL in db is invalid."))
s.try_into()
.map_err(|_| Error::bad_database("Avatar URL in db is invalid."))
})
.transpose()
}
/// Sets a new avatar_url or removes it if avatar_url is None.
#[tracing::instrument(skip(self, user_id, avatar_url))]
pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<MxcUri>) -> Result<()> {
pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<Box<MxcUri>>) -> Result<()> {
if let Some(avatar_url) = avatar_url {
self.userid_avatarurl
.insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?;
@ -245,7 +303,7 @@ impl Users {
.expect("Device::to_string never fails."),
)?;
self.set_token(user_id, &device_id, token)?;
self.set_token(user_id, device_id, token)?;
Ok(())
}
@ -294,7 +352,7 @@ impl Users {
.scan_prefix(prefix)
.map(|(bytes, _)| {
Ok(utils::string_from_bytes(
&bytes
bytes
.rsplit(|&b| b == 0xff)
.next()
.ok_or_else(|| Error::bad_database("UserDevice ID in db is invalid."))?,
@ -342,7 +400,7 @@ impl Users {
user_id: &UserId,
device_id: &DeviceId,
one_time_key_key: &DeviceKeyId,
one_time_key_value: &OneTimeKey,
one_time_key_value: &Raw<OneTimeKey>,
globals: &super::globals::Globals,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
@ -357,7 +415,7 @@ impl Users {
// TODO: Use DeviceKeyId::to_string when it's available (and update everything,
// because there are no wrapping quotation marks anymore)
key.extend_from_slice(
&serde_json::to_string(one_time_key_key)
serde_json::to_string(one_time_key_key)
.expect("DeviceKeyId::to_string always works")
.as_bytes(),
);
@ -368,7 +426,7 @@ impl Users {
)?;
self.userid_lastonetimekeyupdate
.insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
.insert(user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
Ok(())
}
@ -376,7 +434,7 @@ impl Users {
#[tracing::instrument(skip(self, user_id))]
pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result<u64> {
self.userid_lastonetimekeyupdate
.get(&user_id.as_bytes())?
.get(user_id.as_bytes())?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@ -392,7 +450,7 @@ impl Users {
device_id: &DeviceId,
key_algorithm: &DeviceKeyAlgorithm,
globals: &super::globals::Globals,
) -> Result<Option<(DeviceKeyId, OneTimeKey)>> {
) -> Result<Option<(Box<DeviceKeyId>, Raw<OneTimeKey>)>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(device_id.as_bytes());
@ -402,7 +460,7 @@ impl Users {
prefix.push(b':');
self.userid_lastonetimekeyupdate
.insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
.insert(user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
self.onetimekeyid_onetimekeys
.scan_prefix(prefix)
@ -442,7 +500,7 @@ impl Users {
.scan_prefix(userdeviceid)
.map(|(bytes, _)| {
Ok::<_, Error>(
serde_json::from_slice::<DeviceKeyId>(
serde_json::from_slice::<Box<DeviceKeyId>>(
&*bytes.rsplit(|&b| b == 0xff).next().ok_or_else(|| {
Error::bad_database("OneTimeKey ID in db is invalid.")
})?,
@ -463,7 +521,7 @@ impl Users {
&self,
user_id: &UserId,
device_id: &DeviceId,
device_keys: &DeviceKeys,
device_keys: &Raw<DeviceKeys>,
rooms: &super::rooms::Rooms,
globals: &super::globals::Globals,
) -> Result<()> {
@ -492,9 +550,9 @@ impl Users {
pub fn add_cross_signing_keys(
&self,
user_id: &UserId,
master_key: &CrossSigningKey,
self_signing_key: &Option<CrossSigningKey>,
user_signing_key: &Option<CrossSigningKey>,
master_key: &Raw<CrossSigningKey>,
self_signing_key: &Option<Raw<CrossSigningKey>>,
user_signing_key: &Option<Raw<CrossSigningKey>>,
rooms: &super::rooms::Rooms,
globals: &super::globals::Globals,
) -> Result<()> {
@ -504,7 +562,12 @@ impl Users {
prefix.push(0xff);
// Master key
let mut master_key_ids = master_key.keys.values();
let mut master_key_ids = master_key
.deserialize()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))?
.keys
.into_values();
let master_key_id = master_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Master key contained no key.",
@ -520,17 +583,22 @@ impl Users {
let mut master_key_key = prefix.clone();
master_key_key.extend_from_slice(master_key_id.as_bytes());
self.keyid_key.insert(
&master_key_key,
&serde_json::to_vec(&master_key).expect("CrossSigningKey::to_vec always works"),
)?;
self.keyid_key
.insert(&master_key_key, master_key.json().get().as_bytes())?;
self.userid_masterkeyid
.insert(user_id.as_bytes(), &master_key_key)?;
// Self-signing key
if let Some(self_signing_key) = self_signing_key {
let mut self_signing_key_ids = self_signing_key.keys.values();
let mut self_signing_key_ids = self_signing_key
.deserialize()
.map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key")
})?
.keys
.into_values();
let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Self signing key contained no key.",
@ -548,8 +616,7 @@ impl Users {
self.keyid_key.insert(
&self_signing_key_key,
&serde_json::to_vec(&self_signing_key)
.expect("CrossSigningKey::to_vec always works"),
self_signing_key.json().get().as_bytes(),
)?;
self.userid_selfsigningkeyid
@ -558,7 +625,14 @@ impl Users {
// User-signing key
if let Some(user_signing_key) = user_signing_key {
let mut user_signing_key_ids = user_signing_key.keys.values();
let mut user_signing_key_ids = user_signing_key
.deserialize()
.map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key")
})?
.keys
.into_values();
let user_signing_key_id = user_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"User signing key contained no key.",
@ -576,8 +650,7 @@ impl Users {
self.keyid_key.insert(
&user_signing_key_key,
&serde_json::to_vec(&user_signing_key)
.expect("CrossSigningKey::to_vec always works"),
user_signing_key.json().get().as_bytes(),
)?;
self.userid_usersigningkeyid
@ -603,10 +676,11 @@ impl Users {
key.push(0xff);
key.extend_from_slice(key_id.as_bytes());
let mut cross_signing_key =
serde_json::from_slice::<serde_json::Value>(&self.keyid_key.get(&key)?.ok_or(
Error::BadRequest(ErrorKind::InvalidParam, "Tried to sign nonexistent key."),
)?)
let mut cross_signing_key: serde_json::Value =
serde_json::from_slice(&self.keyid_key.get(&key)?.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Tried to sign nonexistent key.",
))?)
.map_err(|_| Error::bad_database("key in keyid_key is invalid."))?;
let signatures = cross_signing_key
@ -614,7 +688,7 @@ impl Users {
.ok_or_else(|| Error::bad_database("key in keyid_key has no signatures field."))?
.as_object_mut()
.ok_or_else(|| Error::bad_database("key in keyid_key has invalid signatures field."))?
.entry(sender_id.clone())
.entry(sender_id.to_owned())
.or_insert_with(|| serde_json::Map::new().into());
signatures
@ -639,7 +713,7 @@ impl Users {
user_or_room_id: &str,
from: u64,
to: Option<u64>,
) -> impl Iterator<Item = Result<UserId>> + 'a {
) -> impl Iterator<Item = Result<Box<UserId>>> + 'a {
let mut prefix = user_or_room_id.as_bytes().to_vec();
prefix.push(0xff);
@ -665,7 +739,7 @@ impl Users {
}
})
.map(|(_, bytes)| {
UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid."))
@ -680,7 +754,7 @@ impl Users {
globals: &super::globals::Globals,
) -> Result<()> {
let count = globals.next_count()?.to_be_bytes();
for room_id in rooms.rooms_joined(&user_id).filter_map(|r| r.ok()) {
for room_id in rooms.rooms_joined(user_id).filter_map(|r| r.ok()) {
// Don't send key updates to unencrypted rooms
if rooms
.room_state_get(&room_id, &EventType::RoomEncryption, "")?
@ -709,7 +783,7 @@ impl Users {
&self,
user_id: &UserId,
device_id: &DeviceId,
) -> Result<Option<DeviceKeys>> {
) -> Result<Option<Raw<DeviceKeys>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(device_id.as_bytes());
@ -726,25 +800,19 @@ impl Users {
&self,
user_id: &UserId,
allowed_signatures: F,
) -> Result<Option<CrossSigningKey>> {
) -> Result<Option<Raw<CrossSigningKey>>> {
self.userid_masterkeyid
.get(user_id.as_bytes())?
.map_or(Ok(None), |key| {
self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
.map_err(|_| {
Error::bad_database("CrossSigningKey in db is invalid.")
})?;
let mut cross_signing_key = serde_json::from_slice::<serde_json::Value>(&bytes)
.map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?;
clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?;
// A user is not allowed to see signatures from users other than himself and
// the target user
cross_signing_key.signatures = cross_signing_key
.signatures
.into_iter()
.filter(|(user, _)| allowed_signatures(user))
.collect();
Ok(Some(cross_signing_key))
Ok(Some(Raw::from_json(
serde_json::value::to_raw_value(&cross_signing_key)
.expect("Value to RawValue serialization"),
)))
})
})
}
@ -754,31 +822,25 @@ impl Users {
&self,
user_id: &UserId,
allowed_signatures: F,
) -> Result<Option<CrossSigningKey>> {
) -> Result<Option<Raw<CrossSigningKey>>> {
self.userid_selfsigningkeyid
.get(user_id.as_bytes())?
.map_or(Ok(None), |key| {
self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
.map_err(|_| {
Error::bad_database("CrossSigningKey in db is invalid.")
})?;
let mut cross_signing_key = serde_json::from_slice::<serde_json::Value>(&bytes)
.map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?;
clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?;
// A user is not allowed to see signatures from users other than himself and
// the target user
cross_signing_key.signatures = cross_signing_key
.signatures
.into_iter()
.filter(|(user, _)| user == user_id || allowed_signatures(user))
.collect();
Ok(Some(cross_signing_key))
Ok(Some(Raw::from_json(
serde_json::value::to_raw_value(&cross_signing_key)
.expect("Value to RawValue serialization"),
)))
})
})
}
#[tracing::instrument(skip(self, user_id))]
pub fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<CrossSigningKey>> {
pub fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<Raw<CrossSigningKey>>> {
self.userid_usersigningkeyid
.get(user_id.as_bytes())?
.map_or(Ok(None), |key| {
@ -961,7 +1023,7 @@ impl Users {
pub fn deactivate_account(&self, user_id: &UserId) -> Result<()> {
// Remove all associated devices
for device_id in self.all_device_ids(user_id) {
self.remove_device(&user_id, &device_id?)?;
self.remove_device(user_id, &device_id?)?;
}
// Set the password to "" to indicate a deactivated account. Hashes will never result in an
@ -972,4 +1034,72 @@ impl Users {
// TODO: Unhook 3PID
Ok(())
}
/// Creates a new sync filter. Returns the filter id.
#[tracing::instrument(skip(self))]
pub fn create_filter(
&self,
user_id: &UserId,
filter: &IncomingFilterDefinition,
) -> Result<String> {
let filter_id = utils::random_string(4);
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(filter_id.as_bytes());
self.userfilterid_filter.insert(
&key,
&serde_json::to_vec(&filter).expect("filter is valid json"),
)?;
Ok(filter_id)
}
#[tracing::instrument(skip(self))]
pub fn get_filter(
&self,
user_id: &UserId,
filter_id: &str,
) -> Result<Option<IncomingFilterDefinition>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(filter_id.as_bytes());
let raw = self.userfilterid_filter.get(&key)?;
if let Some(raw) = raw {
serde_json::from_slice(&raw)
.map_err(|_| Error::bad_database("Invalid filter event in db."))
} else {
Ok(None)
}
}
}
/// Ensure that a user only sees signatures from themselves and the target user
fn clean_signatures<F: Fn(&UserId) -> bool>(
cross_signing_key: &mut serde_json::Value,
user_id: &UserId,
allowed_signatures: F,
) -> Result<(), Error> {
if let Some(signatures) = cross_signing_key
.get_mut("signatures")
.and_then(|v| v.as_object_mut())
{
// Don't allocate for the full size of the current signatures, but require
// at most one resize if nothing is dropped
let new_capacity = signatures.len() / 2;
for (user, signature) in
mem::replace(signatures, serde_json::Map::with_capacity(new_capacity))
{
let id = <&UserId>::try_from(user.as_str())
.map_err(|_| Error::bad_database("Invalid user ID in database."))?;
if id == user_id || allowed_signatures(id) {
signatures.insert(user, signature);
}
}
}
Ok(())
}

23
src/error.rs
View file

@ -8,6 +8,9 @@ use ruma::{
use thiserror::Error;
use tracing::warn;
#[cfg(feature = "persy")]
use persy::PersyError;
#[cfg(feature = "conduit_bin")]
use {
crate::RumaResponse,
@ -20,7 +23,7 @@ use {
tracing::error,
};
pub type Result<T> = std::result::Result<T, Error>;
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Error, Debug)]
pub enum Error {
@ -36,9 +39,18 @@ pub enum Error {
#[from]
source: rusqlite::Error,
},
#[cfg(feature = "persy")]
#[error("There was a problem with the connection to the persy database.")]
PersyError { source: PersyError },
#[cfg(feature = "heed")]
#[error("There was a problem with the connection to the heed database: {error}")]
HeedError { error: String },
#[cfg(feature = "rocksdb")]
#[error("There was a problem with the connection to the rocksdb database: {source}")]
RocksDbError {
#[from]
source: rocksdb::Error,
},
#[error("Could not generate an image.")]
ImageError {
#[from]
@ -136,3 +148,12 @@ where
self.to_response().respond_to(r)
}
}
#[cfg(feature = "persy")]
impl<T: Into<PersyError>> From<persy::PE<T>> for Error {
fn from(err: persy::PE<T>) -> Self {
Error::PersyError {
source: err.error().into(),
}
}
}

20
src/lib.rs
View file

@ -1,21 +1,31 @@
#![warn(
rust_2018_idioms,
unused_qualifications,
clippy::cloned_instead_of_copied,
clippy::str_to_string
)]
#![allow(clippy::suspicious_else_formatting)]
#![deny(clippy::dbg_macro)]
pub mod appservice_server;
pub mod client_server;
use std::ops::Deref;
mod config;
mod database;
mod error;
mod pdu;
mod ruma_wrapper;
pub mod server_server;
mod utils;
pub use database::{Config, Database};
pub mod appservice_server;
pub mod client_server;
pub mod server_server;
pub use config::Config;
pub use database::Database;
pub use error::{Error, Result};
pub use pdu::PduEvent;
pub use rocket::Config as RocketConfig;
pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
use std::ops::Deref;
pub struct State<'r, T: Send + Sync + 'static>(pub &'r T);

107
src/main.rs
View file

@ -1,28 +1,16 @@
#![warn(rust_2018_idioms)]
#![warn(
rust_2018_idioms,
unused_qualifications,
clippy::cloned_instead_of_copied,
clippy::str_to_string
)]
#![allow(clippy::suspicious_else_formatting)]
#![deny(clippy::dbg_macro)]
pub mod appservice_server;
pub mod client_server;
pub mod server_server;
mod database;
mod error;
mod pdu;
mod ruma_wrapper;
mod utils;
use std::sync::Arc;
use database::Config;
pub use database::Database;
pub use error::{Error, Result};
use maplit::hashset;
use opentelemetry::trace::{FutureExt, Tracer};
pub use pdu::PduEvent;
pub use rocket::State;
use ruma::api::client::error::ErrorKind;
pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
use rocket::{
catch, catchers,
figment::{
@ -31,9 +19,20 @@ use rocket::{
},
routes, Request,
};
use ruma::api::client::error::ErrorKind;
use tokio::sync::RwLock;
use tracing_subscriber::{prelude::*, EnvFilter};
pub use conduit::*; // Re-export everything from the library crate
pub use rocket::State;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
use tikv_jemallocator::Jemalloc;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<rocket::Build> {
rocket::custom(config)
.manage(data)
@ -96,6 +95,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
client_server::create_typing_event_route,
client_server::create_room_route,
client_server::redact_event_route,
client_server::report_event_route,
client_server::create_alias_route,
client_server::delete_alias_route,
client_server::get_alias_route,
@ -130,6 +130,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
client_server::send_event_to_device_route,
client_server::get_media_config_route,
client_server::create_content_route,
client_server::get_content_as_filename_route,
client_server::get_content_route,
client_server::get_content_thumbnail_route,
client_server::get_devices_route,
@ -184,9 +185,6 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
#[rocket::main]
async fn main() {
// Force log level off, so we can use our own logger
std::env::set_var("CONDUIT_LOG_LEVEL", "off");
let raw_config =
Figment::from(default_config())
.merge(
@ -197,18 +195,27 @@ async fn main() {
)
.merge(Env::prefixed("CONDUIT_").global());
std::env::set_var("RUST_LOG", "warn");
let config = raw_config
.extract::<Config>()
.expect("It looks like your config is invalid. Please take a look at the error");
let config = match raw_config.extract::<Config>() {
Ok(s) => s,
Err(e) => {
eprintln!("It looks like your config is invalid. The following error occured while parsing it: {}", e);
std::process::exit(1);
}
};
let start = async {
config.warn_deprecated();
let db = Database::load_or_create(&config)
.await
.expect("config is valid");
let db = match Database::load_or_create(&config).await {
Ok(db) => db,
Err(e) => {
eprintln!(
"The database couldn't be loaded or created. The following error occured: {}",
e
);
std::process::exit(1);
}
};
let rocket = setup_rocket(raw_config, Arc::clone(&db))
.ignite()
@ -233,8 +240,6 @@ async fn main() {
println!("exporting");
opentelemetry::global::shutdown_tracer_provider();
} else {
std::env::set_var("RUST_LOG", &config.log);
let registry = tracing_subscriber::Registry::default();
if config.tracing_flame {
let (flame_layer, _guard) =
@ -248,7 +253,7 @@ async fn main() {
start.await;
} else {
let fmt_layer = tracing_subscriber::fmt::Layer::new();
let filter_layer = EnvFilter::try_from_default_env()
let filter_layer = EnvFilter::try_new(&config.log)
.or_else(|_| EnvFilter::try_new("info"))
.unwrap();
@ -288,28 +293,26 @@ fn bad_json_catcher() -> Result<()> {
}
fn default_config() -> rocket::Config {
let mut config = rocket::Config::release_default();
use rocket::config::{LogLevel, Shutdown, Sig};
{
let mut shutdown = &mut config.shutdown;
rocket::Config {
// Disable rocket's logging to get only tracing-subscriber's log output
log_level: LogLevel::Off,
shutdown: Shutdown {
// Once shutdown is triggered, this is the amount of seconds before rocket
// will forcefully start shutting down connections, this gives enough time to /sync
// requests and the like (which havent gotten the memo, somehow) to still complete gracefully.
grace: 35,
#[cfg(unix)]
{
use rocket::config::Sig;
// After the grace period, rocket starts shutting down connections, and waits at least this
// many seconds before forcefully shutting all of them down.
mercy: 10,
shutdown.signals.insert(Sig::Term);
shutdown.signals.insert(Sig::Int);
}
#[cfg(unix)]
signals: hashset![Sig::Term, Sig::Int],
// Once shutdown is triggered, this is the amount of seconds before rocket
// will forcefully start shutting down connections, this gives enough time to /sync
// requests and the like (which havent gotten the memo, somehow) to still complete gracefully.
shutdown.grace = 35;
// After the grace period, rocket starts shutting down connections, and waits at least this
// many seconds before forcefully shutting all of them down.
shutdown.mercy = 10;
..Shutdown::default()
},
..rocket::Config::release_default()
}
config
}

159
src/pdu.rs
View file

@ -1,45 +1,54 @@
use crate::Error;
use ruma::{
events::{
pdu::EventHash, room::member::MemberEventContent, AnyEphemeralRoomEvent,
AnyInitialStateEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent,
AnySyncStateEvent, EventType, StateEvent,
room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent,
AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
},
serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName,
ServerSigningKeyId, UInt, UserId,
state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom};
use serde_json::{
json,
value::{to_raw_value, RawValue as RawJsonValue},
};
use std::{cmp::Ordering, collections::BTreeMap, sync::Arc};
use tracing::warn;
/// Content hashes of a PDU.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct EventHash {
/// The SHA-256 hash.
pub sha256: String,
}
#[derive(Clone, Deserialize, Serialize, Debug)]
pub struct PduEvent {
pub event_id: EventId,
pub room_id: RoomId,
pub sender: UserId,
pub event_id: Arc<EventId>,
pub room_id: Box<RoomId>,
pub sender: Box<UserId>,
pub origin_server_ts: UInt,
#[serde(rename = "type")]
pub kind: EventType,
pub content: serde_json::Value,
pub content: Box<RawJsonValue>,
#[serde(skip_serializing_if = "Option::is_none")]
pub state_key: Option<String>,
pub prev_events: Vec<EventId>,
pub prev_events: Vec<Arc<EventId>>,
pub depth: UInt,
pub auth_events: Vec<EventId>,
pub auth_events: Vec<Arc<EventId>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub redacts: Option<EventId>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub unsigned: BTreeMap<String, serde_json::Value>,
pub redacts: Option<Arc<EventId>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unsigned: Option<Box<RawJsonValue>>,
pub hashes: EventHash,
pub signatures: BTreeMap<Box<ServerName>, BTreeMap<ServerSigningKeyId, String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub signatures: Option<Box<RawJsonValue>>, // BTreeMap<Box<ServerName>, BTreeMap<ServerSigningKeyId, String>>
}
impl PduEvent {
#[tracing::instrument(skip(self))]
pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> {
self.unsigned.clear();
self.unsigned = None;
let allowed: &[&str] = match self.kind {
EventType::RoomMember => &["membership"],
@ -59,10 +68,9 @@ impl PduEvent {
_ => &[],
};
let old_content = self
.content
.as_object_mut()
.ok_or_else(|| Error::bad_database("PDU in db has invalid content."))?;
let mut old_content: BTreeMap<String, serde_json::Value> =
serde_json::from_str(self.content.get())
.map_err(|_| Error::bad_database("PDU in db has invalid content."))?;
let mut new_content = serde_json::Map::new();
@ -72,12 +80,23 @@ impl PduEvent {
}
}
self.unsigned.insert(
"redacted_because".to_owned(),
serde_json::to_value(reason).expect("to_value(PduEvent) always works"),
);
self.unsigned = Some(to_raw_value(&json!({
"redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works")
})).expect("to string always works"));
self.content = new_content.into();
self.content = to_raw_value(&new_content).expect("to string always works");
Ok(())
}
pub fn remove_transaction_id(&mut self) -> crate::Result<()> {
if let Some(unsigned) = &self.unsigned {
let mut unsigned: BTreeMap<String, Box<RawJsonValue>> =
serde_json::from_str(unsigned.get())
.map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?;
unsigned.remove("transaction_id");
self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid"));
}
Ok(())
}
@ -192,7 +211,7 @@ impl PduEvent {
}
#[tracing::instrument(skip(self))]
pub fn to_member_event(&self) -> Raw<StateEvent<MemberEventContent>> {
pub fn to_member_event(&self) -> Raw<StateEvent<RoomMemberEventContent>> {
let json = json!({
"content": self.content,
"type": self.kind,
@ -212,7 +231,7 @@ impl PduEvent {
#[tracing::instrument]
pub fn convert_to_outgoing_federation_event(
mut pdu_json: CanonicalJsonObject,
) -> Raw<ruma::events::pdu::Pdu> {
) -> Box<RawJsonValue> {
if let Some(unsigned) = pdu_json
.get_mut("unsigned")
.and_then(|val| val.as_object_mut())
@ -229,10 +248,7 @@ impl PduEvent {
// )
// .expect("Raw::from_value always works")
serde_json::from_value::<Raw<_>>(
serde_json::to_value(pdu_json).expect("CanonicalJson is valid serde_json::Value"),
)
.expect("Raw::from_value always works")
to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value")
}
pub fn from_id_val(
@ -240,7 +256,7 @@ impl PduEvent {
mut json: CanonicalJsonObject,
) -> Result<Self, serde_json::Error> {
json.insert(
"event_id".to_string(),
"event_id".to_owned(),
CanonicalJsonValue::String(event_id.as_str().to_owned()),
);
@ -249,7 +265,9 @@ impl PduEvent {
}
impl state_res::Event for PduEvent {
fn event_id(&self) -> &EventId {
type Id = Arc<EventId>;
fn event_id(&self) -> &Self::Id {
&self.event_id
}
@ -260,40 +278,34 @@ impl state_res::Event for PduEvent {
fn sender(&self) -> &UserId {
&self.sender
}
fn kind(&self) -> EventType {
self.kind.clone()
fn event_type(&self) -> &EventType {
&self.kind
}
fn content(&self) -> serde_json::Value {
self.content.clone()
fn content(&self) -> &RawJsonValue {
&self.content
}
fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch {
MilliSecondsSinceUnixEpoch(self.origin_server_ts)
}
fn state_key(&self) -> Option<String> {
self.state_key.clone()
fn state_key(&self) -> Option<&str> {
self.state_key.as_deref()
}
fn prev_events(&self) -> Vec<EventId> {
self.prev_events.to_vec()
fn prev_events(&self) -> Box<dyn DoubleEndedIterator<Item = &Self::Id> + '_> {
Box::new(self.prev_events.iter())
}
fn depth(&self) -> &UInt {
&self.depth
fn auth_events(&self) -> Box<dyn DoubleEndedIterator<Item = &Self::Id> + '_> {
Box::new(self.auth_events.iter())
}
fn auth_events(&self) -> Vec<EventId> {
self.auth_events.to_vec()
}
fn redacts(&self) -> Option<&EventId> {
fn redacts(&self) -> Option<&Self::Id> {
self.redacts.as_ref()
}
fn hashes(&self) -> &EventHash {
&self.hashes
}
fn signatures(&self) -> BTreeMap<Box<ServerName>, BTreeMap<ruma::ServerSigningKeyId, String>> {
self.signatures.clone()
}
fn unsigned(&self) -> &BTreeMap<String, serde_json::Value> {
&self.unsigned
}
}
// These impl's allow us to dedup state snapshots when resolving state
@ -319,19 +331,20 @@ impl Ord for PduEvent {
///
/// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap<String, CanonicalJsonValue>`.
pub(crate) fn gen_event_id_canonical_json(
pdu: &Raw<ruma::events::pdu::Pdu>,
) -> crate::Result<(EventId, CanonicalJsonObject)> {
let value = serde_json::from_str(pdu.json().get()).map_err(|e| {
pdu: &RawJsonValue,
) -> crate::Result<(Box<EventId>, CanonicalJsonObject)> {
let value = serde_json::from_str(pdu.get()).map_err(|e| {
warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
Error::BadServerResponse("Invalid PDU in server response")
})?;
let event_id = EventId::try_from(&*format!(
let event_id = format!(
"${}",
// Anything higher than version3 behaves the same
ruma::signatures::reference_hash(&value, &RoomVersionId::Version6)
ruma::signatures::reference_hash(&value, &RoomVersionId::V6)
.expect("ruma can calculate reference hashes")
))
)
.try_into()
.expect("ruma's reference hashes are valid event ids");
Ok((event_id, value))
@ -342,22 +355,8 @@ pub(crate) fn gen_event_id_canonical_json(
pub struct PduBuilder {
#[serde(rename = "type")]
pub event_type: EventType,
pub content: serde_json::Value,
pub content: Box<RawJsonValue>,
pub unsigned: Option<BTreeMap<String, serde_json::Value>>,
pub state_key: Option<String>,
pub redacts: Option<EventId>,
}
/// Direct conversion prevents loss of the empty `state_key` that ruma requires.
impl From<AnyInitialStateEvent> for PduBuilder {
fn from(event: AnyInitialStateEvent) -> Self {
Self {
event_type: EventType::from(event.event_type()),
content: serde_json::value::to_value(event.content())
.expect("AnyStateEventContent came from JSON and can thus turn back into JSON."),
unsigned: None,
state_key: Some(event.state_key().to_owned()),
redacts: None,
}
}
pub redacts: Option<Arc<EventId>>,
}

29
src/ruma_wrapper.rs
View file

@ -20,7 +20,6 @@ use {
},
ruma::api::{AuthScheme, IncomingRequest},
std::collections::BTreeMap,
std::convert::TryFrom,
std::io::Cursor,
tracing::{debug, warn},
};
@ -29,7 +28,7 @@ use {
/// first.
pub struct Ruma<T: Outgoing> {
pub body: T::Incoming,
pub sender_user: Option<UserId>,
pub sender_user: Option<Box<UserId>>,
pub sender_device: Option<Box<DeviceId>>,
pub sender_servername: Option<Box<ServerName>>,
// This is None when body is not a valid string
@ -66,7 +65,7 @@ where
let limit = db.globals.max_request_size();
let mut handle = data.open(ByteUnit::Byte(limit.into()));
let mut body = Vec::new();
if let Err(_) = handle.read_to_end(&mut body).await {
if handle.read_to_end(&mut body).await.is_err() {
// Client disconnected
// Missing Token
return Failure((Status::new(582), ()));
@ -86,7 +85,7 @@ where
registration
.get("as_token")
.and_then(|as_token| as_token.as_str())
.map_or(false, |as_token| token.as_deref() == Some(as_token))
.map_or(false, |as_token| token == Some(as_token))
}) {
match metadata.authentication {
AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
@ -103,8 +102,7 @@ where
.unwrap()
},
|string| {
UserId::try_from(string.expect("parsing to string always works"))
.unwrap()
UserId::parse(string.expect("parsing to string always works")).unwrap()
},
);
@ -123,7 +121,7 @@ where
match metadata.authentication {
AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
if let Some(token) = token {
match db.users.find_from_token(&token).unwrap() {
match db.users.find_from_token(token).unwrap() {
// Unknown Token
None => return Failure((Status::new(581), ())),
Some((user_id, device_id)) => (
@ -171,7 +169,7 @@ where
}
};
let origin = match Box::<ServerName>::try_from(origin_str) {
let origin = match ServerName::parse(origin_str) {
Ok(s) => s,
_ => {
warn!(
@ -298,14 +296,11 @@ where
.and_then(|auth| auth.get("session"))
.and_then(|session| session.as_str())
.and_then(|session| {
db.uiaa
.get_uiaa_request(
&user_id,
&sender_device.clone().unwrap_or_else(|| "".into()),
session,
)
.ok()
.flatten()
db.uiaa.get_uiaa_request(
&user_id,
&sender_device.clone().unwrap_or_else(|| "".into()),
session,
)
})
{
for (key, value) in initial_request {
@ -344,7 +339,7 @@ impl<T: Outgoing> Deref for Ruma<T> {
}
/// This struct converts ruma responses into rocket http responses.
pub type ConduitResult<T> = std::result::Result<RumaResponse<T>, Error>;
pub type ConduitResult<T> = Result<RumaResponse<T>, Error>;
pub fn response<T: OutgoingResponse>(response: RumaResponse<T>) -> response::Result<'static> {
let http_response = response

929
src/server_server.rs
View file

File diff suppressed because it is too large Load diff

3
src/utils.rs
View file

@ -4,7 +4,6 @@ use rand::prelude::*;
use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject};
use std::{
cmp,
convert::TryInto,
str::FromStr,
time::{SystemTime, UNIX_EPOCH},
};
@ -123,7 +122,7 @@ pub fn deserialize_from_str<
E: std::fmt::Display,
>(
deserializer: D,
) -> std::result::Result<T, D::Error> {
) -> Result<T, D::Error> {
struct Visitor<T: FromStr<Err = E>, E>(std::marker::PhantomData<T>);
impl<'de, T: FromStr<Err = Err>, Err: std::fmt::Display> serde::de::Visitor<'de>
for Visitor<T, Err>

101
tests/client-element-web/test-element-web-registration.js
View file

@ -1,101 +0,0 @@
const puppeteer = require('puppeteer');
run().then(() => console.log('Done')).catch(error => {
console.error("Registration test failed.");
console.error("There might be a screenshot of the failure in the artifacts.\n");
console.error(error);
process.exit(111);
});
async function run() {
const elementUrl = process.argv[process.argv.length - 2];
console.debug("Testing registration with ElementWeb hosted at "+ elementUrl);
const homeserverUrl = process.argv[process.argv.length - 1];
console.debug("Homeserver url: "+ homeserverUrl);
const username = "testuser" + String(Math.floor(Math.random() * 100000));
const password = "testpassword" + String(Math.floor(Math.random() * 100000));
console.debug("Testuser for this run:\n User: " + username + "\n Password: " + password);
const browser = await puppeteer.launch({
headless: true, args: [
"--no-sandbox"
]
});
const page = await browser.newPage();
await page.goto(elementUrl);
await page.screenshot({ path: '01-element-web-opened.png' });
console.debug("Click [Create Account] button");
await page.waitForSelector('a.mx_ButtonCreateAccount');
await page.click('a.mx_ButtonCreateAccount');
await page.screenshot({ path: '02-clicked-create-account-button.png' });
// The webapp should have loaded right now, if anything takes more than 5 seconds, something probably broke
page.setDefaultTimeout(5000);
console.debug("Click [Edit] to switch homeserver");
await page.waitForSelector('div.mx_ServerPicker_change');
await page.click('div.mx_ServerPicker_change');
await page.screenshot({ path: '03-clicked-edit-homeserver-button.png' });
console.debug("Type in local homeserver url");
await page.waitForSelector('input#mx_homeserverInput');
await page.click('input#mx_homeserverInput');
await page.click('input#mx_homeserverInput');
await page.keyboard.type(homeserverUrl);
await page.screenshot({ path: '04-typed-in-homeserver.png' });
console.debug("[Continue] with changed homeserver");
await page.waitForSelector("div.mx_ServerPickerDialog_continue");
await page.click('div.mx_ServerPickerDialog_continue');
await page.screenshot({ path: '05-back-to-enter-user-credentials.png' });
console.debug("Type in username");
await page.waitForSelector("input#mx_RegistrationForm_username");
await page.click('input#mx_RegistrationForm_username');
await page.keyboard.type(username);
await page.screenshot({ path: '06-typed-in-username.png' });
console.debug("Type in password");
await page.waitForSelector("input#mx_RegistrationForm_password");
await page.click('input#mx_RegistrationForm_password');
await page.keyboard.type(password);
await page.screenshot({ path: '07-typed-in-password-once.png' });
console.debug("Type in password again");
await page.waitForSelector("input#mx_RegistrationForm_passwordConfirm");
await page.click('input#mx_RegistrationForm_passwordConfirm');
await page.keyboard.type(password);
await page.screenshot({ path: '08-typed-in-password-twice.png' });
console.debug("Click on [Register] to finish the account creation");
await page.waitForSelector("input.mx_Login_submit");
await page.click('input.mx_Login_submit');
await page.screenshot({ path: '09-clicked-on-register-button.png' });
// Waiting for the app to login can take some time, so be patient.
page.setDefaultTimeout(10000);
console.debug("Wait for chat window to show up");
await page.waitForSelector("div.mx_HomePage_default_buttons");
console.debug("Apparently the registration worked.");
await page.screenshot({ path: '10-logged-in-homescreen.png' });
// Close the browser and exit the script
await browser.close();
}

1
tests/sytest/sytest-whitelist
View file

@ -510,3 +510,4 @@ remote user can join room with version 5
remote user can join room with version 6
setting 'm.room.name' respects room powerlevel
setting 'm.room.power_levels' respects room powerlevel
Federation publicRoom Name/topic keys are correct