pax_global_header00006660000000000000000000000064141451647700014523gustar00rootroot0000000000000052 comment=cf27fce07798c9efeea901ddee4211574840024d sydent-2.5.1/000077500000000000000000000000001414516477000130365ustar00rootroot00000000000000sydent-2.5.1/.dockerignore000066400000000000000000000000621414516477000155100ustar00rootroot00000000000000*.pyc .idea/ .vscode/ sydent.conf sydent.db .git sydent-2.5.1/.github/000077500000000000000000000000001414516477000143765ustar00rootroot00000000000000sydent-2.5.1/.github/CODEOWNERS000066400000000000000000000001641414516477000157720ustar00rootroot00000000000000# Automatically request reviews from the synapse-core team when a pull request comes in. * @matrix-org/synapse-core sydent-2.5.1/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000014341414516477000202010ustar00rootroot00000000000000### Pull Request Checklist * [ ] Pull request includes a [changelog file](https://github.com/matrix-org/sydent/blob/main/CONTRIBUTING.md#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fix a bug that prevented receiving messages from other servers." instead of "Move X method from `EventStore` to `EventWorkerStore`.". - Include 'Contributed by *Your Name*.' or 'Contributed by @*your-github-username*.' — unless you would prefer not to be credited in the changelog. - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. * [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off) sydent-2.5.1/.github/workflows/000077500000000000000000000000001414516477000164335ustar00rootroot00000000000000sydent-2.5.1/.github/workflows/changelog_check.yml000066400000000000000000000010011414516477000222320ustar00rootroot00000000000000name: Changelog on: [pull_request] jobs: check-newsfile: if: ${{ github.base_ref == 'main' || contains(github.base_ref, 'release-') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 ref: ${{github.event.pull_request.head.sha}} - uses: actions/setup-python@v2 with: python-version: "3.7" - run: python -m pip install towncrier - run: "scripts-dev/check_newsfragment.sh ${{ github.event.number }}" sydent-2.5.1/.github/workflows/mypy.yml000066400000000000000000000004471414516477000201610ustar00rootroot00000000000000name: Mypy on: pull_request: push: branches: ["main"] jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.6' - run: python -m pip install -e .[dev] - run: mypy sydent-2.5.1/.github/workflows/pipeline.yml000066400000000000000000000030361414516477000207650ustar00rootroot00000000000000name: Linting and tests on: pull_request: push: branches: ["main"] jobs: check-code-style: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.6' - run: python -m pip install -e .[dev] # Please keep this section up to date with `scripts-dev/lint.sh`. # TODO: could just run the lint script directly? - run: black --check --diff sydent/ stubs/ tests/ matrix_is_test/ scripts/ setup.py - run: flake8 sydent/ tests/ matrix_is_test/ scripts/ setup.py - run: flake8 stubs/ --ignore E301,E302,E305,E701,E704 - run: isort --check-only --diff sydent/ stubs/ tests/ matrix_is_test/ scripts/ setup.py run-unit-tests: needs: [check-code-style] runs-on: ubuntu-latest strategy: matrix: python-version: ['3.6', '3.x'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - run: python -m pip install -e .[dev] - run: trial tests run-matrix-is-tests: needs: [check-code-style] runs-on: ubuntu-latest strategy: matrix: python-version: ['3.6', '3.x'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - run: python -m pip install -e . - run: pip install git+https://github.com/matrix-org/matrix-is-tester.git - run: trial matrix_is_tester sydent-2.5.1/.gitignore000066400000000000000000000003221414516477000150230ustar00rootroot00000000000000# Development files *.pyc .idea/ .vscode/ *.iml _trial_temp _trial_temp.lock *.egg *.egg-info .python-version .mypy_cache /env # Runtime files /sydent.conf /sydent.db /sydent.pid /matrix_is_test/sydent.stderr sydent-2.5.1/CHANGELOG.md000066400000000000000000000525021414516477000146530ustar00rootroot00000000000000Sydent 2.5.1 (2021-11-17) ========================= This release fixes a bug in handling verification for third party IDs if requested via the deprecated `/api/v1/` endpoint. The other changes are all designed to improve error handling, and make Sydent's logging have a higher signal-to-noise ratio. Features -------- - Return HTTP 400 Bad Request rather than HTTP 500 Internal Server Error if `/store-invite` is given an invalid email address. ([\#464](https://github.com/matrix-org/sydent/issues/464)) Bugfixes -------- - __Fix a bug introduced in Sydent 2.5.0 where requests to validate an email or phone number would fail with an HTTP 500 Internal Server Error if arguments were given as a query string or as a www-form-urlencoded body. ([\#461](https://github.com/matrix-org/sydent/issues/461), [\#462](https://github.com/matrix-org/sydent/issues/462))__ Internal Changes ---------------- - Improve exception logging in `asyncjsonwrap` for better Sentry reports. ([\#455](https://github.com/matrix-org/sydent/issues/455)) - Handle federation request failures in `/request` explicitly, to reduce Sentry noise. ([\#456](https://github.com/matrix-org/sydent/issues/456)) - Log a warning (not an error) when we refuse to send an SMS to an unsupported country. ([\#459](https://github.com/matrix-org/sydent/issues/459)) - Demote a failure to parse JSON from homeservers in `/register` from an error to a warning. ([\#463](https://github.com/matrix-org/sydent/issues/463)) - Handle errors to contact homeservers in `/unbind`. This returns a better error message and reduces Sentry spam. ([\#466](https://github.com/matrix-org/sydent/issues/466)) - Log failures to send SMS as exceptions, not errors (to better debug in Sentry). ([\#467](https://github.com/matrix-org/sydent/issues/467)) Sydent 2.5.0 (2021-11-03) ========================= This release [deprecates `.eml` templates](https://github.com/matrix-org/sydent/issues/395) in favour of Jinja 2 `.eml.j2` templates. See the [documentation](https://github.com/matrix-org/sydent/blob/main/docs/templates.md#template-formats) for more details. Features -------- - __Support the stable `room_type` field for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288).__ ([\#437](https://github.com/matrix-org/sydent/issues/437)) Bugfixes -------- - __Fix a bug which could cause SMS sending to fail silently.__ ([\#412](https://github.com/matrix-org/sydent/issues/412)) - Fix a bug introduced in v2.4.0 that caused association unbindings to fail with an internal server error. ([\#397](https://github.com/matrix-org/sydent/issues/397)) - Fix an issue which could cause new local associations to be replicated multiple times to peers. ([\#400](https://github.com/matrix-org/sydent/issues/400)) - Fix an issue where `obey_x_forwarded_for` was not being honoured. ([\#403](https://github.com/matrix-org/sydent/issues/403)) - Fix misleading logging and potential TypeErrors related to replication ports in Sydent's database. ([\#420](https://github.com/matrix-org/sydent/issues/420)) - Fix a bug introduced in v2.0.0 where requesting `GET` from `/identity/api/v1/validate/msisdn/submitToken` or `/identity/v2/validate/msisdn/submitToken` would fail with an internal server error. ([\#445](https://github.com/matrix-org/sydent/issues/445)) - Fix `/v2/account/logout` to return HTTP 400 BAD REQUEST instead of 200 OK if a token was not provided. ([\#447](https://github.com/matrix-org/sydent/issues/447)) - Fix a long-standing spec compliance bug where the response to `POST /identity/{api/v1,v2}/3pid/unbind` was `null`, not `{}`. ([\#449](https://github.com/matrix-org/sydent/issues/449)) Improved Documentation ---------------------- - Fix the documentation around the command line arguments for the email address migration script. ([\#392](https://github.com/matrix-org/sydent/issues/392)) - Add documentation on writing templates. Deprecate .eml templates. ([\#395](https://github.com/matrix-org/sydent/issues/395)) Internal Changes ---------------- - __Improve type annotations throughout Sydent. Sydent now passes `mypy --strict`.__ ([\#414](https://github.com/matrix-org/sydent/issues/414) and others). - Extend the changelog check so that it checks for the correct pull request number being used. ([\#382](https://github.com/matrix-org/sydent/issues/382)) - Move the configuration file handling code into a separate module. ([\#385](https://github.com/matrix-org/sydent/issues/385), [\#405](https://github.com/matrix-org/sydent/issues/405)) - Add a primitive contributing guide and tweak the pull request template. ([\#393](https://github.com/matrix-org/sydent/issues/393)) - Run mypy on the sydent package as part of CI. ([\#416](https://github.com/matrix-org/sydent/issues/416)) - Configure @matrix-org/synapse-core to be the code owner for the repository. ([\#436](https://github.com/matrix-org/sydent/issues/436)) - Run linters over stub files. ([\#441](https://github.com/matrix-org/sydent/issues/441), [\#450](https://github.com/matrix-org/sydent/issues/450)) - Include Sydent's version number (and git commit hash if available) when reporting to Sentry. ([\#453](https://github.com/matrix-org/sydent/issues/453), [\#454](https://github.com/matrix-org/sydent/issues/454)) Sydent 2.4.6 (2021-10-08) ========================= Bugfixes -------- - Fix a long-standing bug with error handling around missing headers when dealing with the OpenMarket API, which could cause the wrong assumption that sending a SMS failed when it didn't. ([\#415](https://github.com/matrix-org/sydent/issues/415)) Sydent 2.4.5 (2021-10-08) ========================= Bugfixes -------- - Fix a long-standing bug in asynchronous code that could cause SMS messages not to be correctly sent. ([\#413](https://github.com/matrix-org/sydent/issues/413)) Sydent 2.4.4 (2021-10-08) ========================= Bugfixes -------- - Fix a bug introduced in v2.4.0 which could cause SMS sending to fail silently. ([\#412](https://github.com/matrix-org/sydent/issues/412)) Sydent 2.4.3 (2021-09-14) ========================= Bugfixes -------- - Fix a bug introduced in v2.4.0 that caused association unbindings to fail with an Internal Server Error. ([\#397](https://github.com/matrix-org/sydent/issues/397)) Sydent 2.4.2 (2021-09-13) ========================= Bugfixes -------- - Fix a bug causing the email address migration script to take a lot of time to run due to inefficient database queries. ([\#396](https://github.com/matrix-org/sydent/issues/396)) Internal Changes ---------------- - Move dev tools from `install_requires` to `extras_require`. ([\#389](https://github.com/matrix-org/sydent/issues/389)) - Run background jobs in `run` rather than in Sydent's constructor. ([\#394](https://github.com/matrix-org/sydent/issues/394)) Sydent 2.4.1 (2021-09-10) ========================= Bugfixes -------- - Fix a bug preventing the email migration script from running while Sydent is already running with Prometheus metrics enabled. ([\#391](https://github.com/matrix-org/sydent/issues/391)) Sydent 2.4.0 (2021-09-09) ========================= **This release drops compatibility with Python 3.5 and older. Python 3.6 and later is required to run Sydent from this version onwards.** **Action required when upgrading**: server administrators should run [the e-mail address migration script](./docs/casefold_migration.md). Features -------- - Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), receiving `room_type` for 3pid invites over the `/store-invite` API and using it in Jinja templates for Space invites. ([\#375](https://github.com/matrix-org/sydent/issues/375)) - Add support for using Jinja2 in e-mail templates. Contributed by @H-Shay. ([\#376](https://github.com/matrix-org/sydent/issues/376)) - Case-fold email addresses when binding to MXIDs or performing look-ups. Contributed by @H-Shay. ([\#374](https://github.com/matrix-org/sydent/issues/374), [\#378](https://github.com/matrix-org/sydent/issues/378), [\#379](https://github.com/matrix-org/sydent/issues/379), [\#386](https://github.com/matrix-org/sydent/issues/386)) Bugfixes -------- - Handle CORS for `GetValidated3pidServlet`. Endpoint `/3pid/getValidated3pid` returns valid CORS headers. ([\#342](https://github.com/matrix-org/sydent/issues/342)) - Use the `web_client_location` parameter in default templates for both text and HTML emails. ([\#380](https://github.com/matrix-org/sydent/issues/380)) Internal Changes ---------------- - Add `/_trial_temp.lock` and `/sydent.pid` to .gitignore. ([\#384](https://github.com/matrix-org/sydent/issues/384)) - Reformat code using Black. Contributed by @H-Shay. ([\#344](https://github.com/matrix-org/sydent/issues/344), [\#369](https://github.com/matrix-org/sydent/issues/369)) - Configure Flake8 and resolve errors. ([\#345](https://github.com/matrix-org/sydent/issues/345), [\#347](https://github.com/matrix-org/sydent/issues/347)) - Add GitHub Actions for unit tests (Python 3.6 and 3.9), matrix_is_tester tests (Python 3.6 and 3.9), towncrier checks and black and flake8 codestyle checks. ([\#346](https://github.com/matrix-org/sydent/issues/346), [\#348](https://github.com/matrix-org/sydent/issues/348)) - Remove support for Python < 3.6. Contributed by @sunweaver. ([\#349](https://github.com/matrix-org/sydent/issues/349), [\#356](https://github.com/matrix-org/sydent/issues/356)) - Bump minimum supported version of Twisted to 18.4.0 and stop calling deprecated APIs. ([\#350](https://github.com/matrix-org/sydent/issues/350)) - Replace deprecated `logging.warn()` method with `logging.warning()`. ([\#351](https://github.com/matrix-org/sydent/issues/351)) - Reformat imports using isort. ([\#352](https://github.com/matrix-org/sydent/issues/352), [\#353](https://github.com/matrix-org/sydent/issues/353)) - Update `.gitignore` to only ignore `sydent.conf` and `sydent.db` if they occur in the project's base folder. ([\#354](https://github.com/matrix-org/sydent/issues/354)) - Add type hints and validate with mypy. ([\#355](https://github.com/matrix-org/sydent/issues/355), [\#357](https://github.com/matrix-org/sydent/issues/357), [\#358](https://github.com/matrix-org/sydent/issues/358), [\#360](https://github.com/matrix-org/sydent/issues/360), [\#361](https://github.com/matrix-org/sydent/issues/361), [\#367](https://github.com/matrix-org/sydent/issues/367), [\#371](https://github.com/matrix-org/sydent/issues/371)) - Convert `inlineCallbacks` to async/await. ([\#364](https://github.com/matrix-org/sydent/issues/364), [\#365](https://github.com/matrix-org/sydent/issues/365), [\#368](https://github.com/matrix-org/sydent/issues/368), [\#372](https://github.com/matrix-org/sydent/issues/372), [\#373](https://github.com/matrix-org/sydent/issues/373)) - Use `mock` module from the standard library. ([\#370](https://github.com/matrix-org/sydent/issues/370)) - Fix email templates to be valid python format strings. ([\#377](https://github.com/matrix-org/sydent/issues/377)) Sydent 2.3.0 (2021-04-15) ========================= **Note**: this will be the last release of Sydent to support Python 3.5 or earlier. Future releases will require at least Python 3.6. Security advisory ----------------- This release contains fixes to the following security issues: - Denial of service attack via disk space or memory exhaustion ([CVE-2021-29430](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2021-29430)). - SSRF due to missing validation of hostnames ([CVE-2021-29431](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2021-29431)). - Malicious users could control the content of invitation emails ([CVE-2021-29432](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2021-29432)). - Denial of service (via resource exhaustion) due to improper input validation ([CVE-2021-29433](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2021-29433)). Although we are not aware of these vulnerabilities being exploited in the wild, Sydent server administrators are advised to update as soon as possible. Note that as well as changes to the package, there are also changes to the default email templates. If any templates have been updated locally, they must also be updated in line with the changes to the defaults for full protection from CVE-2021-29432. Features -------- - Accept an optional `web_client_location` argument to the invite endpoint which allows customisation of the email template. ([\#326](https://github.com/matrix-org/sydent/issues/326)) - Move templates to a per-brand subdirectory of `/res`. Add `templates.path` and `brand.default` config options. ([\#328](https://github.com/matrix-org/sydent/issues/328)) Bugfixes -------- - Fix a regression in v2.2.0 where the wrong characters would be obfuscated in a 3pid invite. ([\#317](https://github.com/matrix-org/sydent/issues/317)) - Fix a long-standing bug where invalid JSON would be accepted over the HTTP interfaces. ([\#337](https://github.com/matrix-org/sydent/issues/337)) - During user registration on the identity server, validate that the MXID returned by the contacted homeserver is valid for that homeserver. ([cc97fff](https://github.com/matrix-org/sydent/commit/cc97fff)) - Ensure that `/v2/` endpoints are correctly authenticated. ([ce04a68](https://github.com/matrix-org/sydent/commit/ce04a68)) - Perform additional validation on the response received when requesting server signing keys. ([07e6da7](https://github.com/matrix-org/sydent/commit/07e6da7)) - Validate the `matrix_server_name` parameter given during user registration. ([9e57334](https://github.com/matrix-org/sydent/commit/9e57334), [8936925](https://github.com/matrix-org/sydent/commit/8936925), [3d531ed](https://github.com/matrix-org/sydent/commit/3d531ed), [0f00412](https://github.com/matrix-org/sydent/commit/0f00412)) - Limit the size of requests received from HTTP clients. ([89071a1](https://github.com/matrix-org/sydent/commit/89071a1), [0523511](https://github.com/matrix-org/sydent/commit/0523511), [f56eee3](https://github.com/matrix-org/sydent/commit/f56eee3)) - Limit the size of responses received from HTTP servers. ([89071a1](https://github.com/matrix-org/sydent/commit/89071a1), [0523511](https://github.com/matrix-org/sydent/commit/0523511), [f56eee3](https://github.com/matrix-org/sydent/commit/f56eee3)) - In invite emails, randomise the multipart boundary, and include MXIDs where available. ([4469d1d](https://github.com/matrix-org/sydent/commit/4469d1d), [6b405a8](https://github.com/matrix-org/sydent/commit/6b405a8), [65a6e91](https://github.com/matrix-org/sydent/commit/65a6e91)) - Perform additional validation on the `client_secret` and `email` parameters to various APIs. ([3175fd3](https://github.com/matrix-org/sydent/commit/3175fd3)) Updates to the Docker image --------------------------- - Base docker image on Debian rather than Alpine Linux. ([\#335](https://github.com/matrix-org/sydent/issues/335)) Internal Changes ---------------- - Fix test logging to allow braces in log output. ([\#318](https://github.com/matrix-org/sydent/issues/318)) - Install prometheus_client in the Docker image. ([\#325](https://github.com/matrix-org/sydent/issues/325)) - Bump the version of signedjson to 1.1.1. ([\#334](https://github.com/matrix-org/sydent/issues/334)) Sydent 2.2.0 (2020-09-11) ========================= Bugfixes -------- - Fix intermittent deadlock in Sentry integration. ([\#312](https://github.com/matrix-org/sydent/issues/312)) Sydent 2.1.0 (2020-09-10) ========================= Features -------- - Add a Dockerfile and allow environment variables `SYDENT_SERVER_NAME`, `SYDENT_PID_FILE` and `SYDENT_DB_PATH` to modify default configuration values. ([\#290](https://github.com/matrix-org/sydent/issues/290)) - Add config options for controlling how email addresses are obfuscated in third party invites. ([\#311](https://github.com/matrix-org/sydent/issues/311)) Bugfixes -------- - Fix a bug in the error handling of 3PID session validation, if the token submitted is incorrect. ([\#296](https://github.com/matrix-org/sydent/issues/296)) - Stop sending the unspecified `success` parameter in responses to `/requestToken` requests. ([\#302](https://github.com/matrix-org/sydent/issues/302)) - Fix a bug causing Sydent to ignore `nextLink` parameters. ([\#303](https://github.com/matrix-org/sydent/issues/303)) - Fix the HTTP status code returned during some error responses. ([\#305](https://github.com/matrix-org/sydent/issues/305)) - Sydent now correctly enforces the valid characters in the `client_secret` parameter used in various endpoints. ([\#309](https://github.com/matrix-org/sydent/issues/309)) Internal Changes ---------------- - Replace instances of Riot with Element. ([\#308](https://github.com/matrix-org/sydent/issues/308)) Sydent 2.0.1 (2020-05-20) ========================= Features -------- - Add a config option to disable deleting invite tokens on bind. ([\#293](https://github.com/matrix-org/sydent/issues/293)) Bugfixes -------- - Fix a bug that prevented Sydent from checking for access tokens in request parameters when running on Python3. ([\#294](https://github.com/matrix-org/sydent/issues/294)) Internal Changes ---------------- - Make replication tests more reliable. ([\#278](https://github.com/matrix-org/sydent/issues/278)) - Add a configuration for towncrier. ([\#295](https://github.com/matrix-org/sydent/issues/295)) Changes in [2.0.0](https://github.com/matrix-org/sydent/releases/tag/v2.0.0) (2020-02-25) ========================================================================================= **Note:** Starting with this release, Sydent releases are available on [PyPI](https://pypi.org/project/matrix-sydent). This means that the recommended method for stable installations is now by using the PyPI project rather than a tarball of the `master` branch of this repository. See [the README](https://github.com/matrix-org/sydent/blob/v2.0.0/README.rst) for more details. **Warning:** This release deprecates v1 APIs for existing endpoints in favour of v2 APIs. Homeserver and client developers are encouraged to migrate their applications to the v2 APIs. See below for more information. Features -------- * Implement the items and MSCs from the [privacy project](https://matrix.org/blog/2019/09/27/privacy-improvements-in-synapse-1-4-and-riot-1-4) targeting identity servers. This introduces v2 APIs for every existing endpoint. v1 APIs are now deprecated and homeserver and client developers are encouraged to migrate their applications to the v2 APIs. * Add Python 3 compatibility to all of the codebase. Python 2 is still supported for now. * Delete stored invites upon successful delivery to a homeserver * Filter out delivered invites when delivering invites to a homeserver upon successful binding * Implement support for authenticating unbind queries by providing a `sid` and a `client_secret`, as per [MSC1915](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1915-unbind-identity-server-param.md) * Add support for Prometheus and Sentry * Handle `.well-known` files when talking to homeservers * Validate `client_secret` parameters according to the Matrix specification * Return 400/404 on incorrect session validation * Add a default 10,000 address limit on v2 `/lookup` (which supports multiple lookups at once) Documentation ------------- * Rewrite part of the README to make it more user-friendly Bugfixes -------- * Fix a bug that would prevent requests to the `/store-invite` endpoint with JSON payloads from being correctly processed * Fix a bug where multiple cleanup tasks would be unnecessary spawned * Fix logging so Sydent doesn't log 3PIDs when processing lookup requests * Fix incorrect HTTP response from `/3pid/getValidated3pid` endpoint on failure. * Prevent a single failure from aborting the federation loop * Fix federation lookups in `/onBind` callbacks * Don't fail the unbind request if the binding doesn't exist * Fix the signing servlet missing a reference to the Sydent object * Fix content types & OPTIONS requests Internal changes ---------------- * Add unit tests to test startup and replication * Add support for testing with `matrix-is-tester` * Remove instances of `setResponseCode(200)` Changes in [1.0.3](https://github.com/matrix-org/sydent/releases/tag/v1.0.3) (2019-05-03) ========================================================================================= * Use trustRoot instead of verify for request verification Security Fixes -------------- * Ensures that authentication tokens are generated using a secure random number generator, ensuring they cannot be predicted by an attacker. Thanks to @opnsec for identifying and responsibly disclosing the issue! * Mitigate an HTML injection bug where an invalid room_id could result in malicious HTML being injected into validation emails. Thanks to @opnsec for identifying and responsibly disclosing this issue too! * Randomise session_ids to avoid leaking info about the total number of identity validations, and whether a given ID has been validated. Thanks to @fs0c131y for this one. * Don't send tracebacks to the browser when errors occur. Changes in [1.0.2](https://github.com/matrix-org/sydent/releases/tag/v1.0.2) (2019-04-18) ========================================================================================= Security Fixes -------------- * Fix for validating malformed email addresses: https://github.com/matrix-org/sydent/commit/3103b65dcfa37a9241dabedba560c4ded6c05ff6 Changes in [1.0.1](https://github.com/matrix-org/sydent/releases/tag/v1.0.1) (2019-04-18) ========================================================================================= Release pointed to wrong commit, fixed by 1.0.2 sydent-2.5.1/CONTRIBUTING.md000066400000000000000000000272561414516477000153030ustar00rootroot00000000000000# Contributing code to Sydent Everyone is welcome to contribute code to Sydent, provided you are willing to license your contributions under the same license as the project itself. In this case, the [Apache Software License v2](LICENSE). ## Set up your development environment ### Create a virtualenv To contribute to Sydent, ensure you have Python 3.7 or newer and then run: ```bash python3 -m venv venv ./venv/bin/pip install -e '.[dev]' ``` This creates an isolated virtual Python environment ("virtualenv") just for use with Sydent, then installs Sydent along with its dependencies, and lastly installs a handful of useful tools If you get `ConnectTimeoutError`, this is caused by slow internet whereby `pip` has a default time out of _15 sec_. You can specify a larger timeout by passing `--timeout 120` to the `pip install` command above. Finally, activate the virtualenv by running: ```bash source ./venv/bin/activate ``` Be sure to do this _every time_ you open a new terminal window for working on Sydent. Activating the venv ensures that any Python commands you run (`pip`, `python`, etc.) use the versions inside your venv, and not your system Python. When you're done, you can close your terminal or run `deactivate` to disable the virtualenv. ### Run the unit tests To make sure everything is working as expected, run the unit tests: ```bash trial tests ``` If you see a message like: ``` ------------------------------------------------------------------------------- Ran 25 tests in 0.324s PASSED (successes=25) ``` Then all is well and you're ready to work! If `trial tests` fails but `python -m twisted.trial tests` succeeds, try ensuring your venv is activated and re-installing using `pip install -e '.[dev]'`, making sure to remember the `-e` flag. ### Run the black-box tests Sydent uses [matrix-is-tester](https://github.com/matrix-org/matrix-is-tester/) to provide black-box testing of compliance with the [Matrix Identity Service API](https://matrix.org/docs/spec/identity_service/latest). (Features that are Sydent-specific belong in unit tests rather than the black-box test suite.) If you have set up a venv using the steps above, you can install `matrix-is-tester` as follows: ``` pip install git+https://github.com/matrix-org/matrix-is-tester.git ``` Now, to run `matrix-is-tester`, execute: ``` trial matrix_is_tester ``` If this doesn't work, ensure that you have first activated your venv and installed Sydent with the editable (`-e`) flag: `pip install -e '.[dev]'`. #### Advanced The steps above are sufficient and describe a clean way to run the black-box tests. However, in the event that you need more control, this subsection provides more information. The `SYDENT_PYTHON` enviroment variable can be set to launch Sydent with a specific python binary: ``` SYDENT_PYTHON=/path/to/python trial matrix_is_tester ``` The `matrix_is_test` directory contains Sydent's launcher for `matrix_is_tester`: this means that Sydent's directory needs to be on the Python path (e.g. `PYTHONPATH=$PYTHONPATH:/path/to/sydent`). ## How to contribute The preferred and easiest way to contribute changes is to fork the relevant project on github, and then [create a pull request]( https://help.github.com/articles/using-pull-requests/) to ask us to pull your changes into our repo. Some other points to follow: * Please base your changes on the `main` branch. * Please follow the [code style requirements](#code-style). * Please include a [changelog entry](#changelog) with each PR. * Please [sign off](#sign-off) your contribution. * Please keep an eye on the pull request for feedback from the [continuous integration system](#continuous-integration-and-testing) and try to fix any errors that come up. * If you need to [update your PR](#updating-your-pull-request), just add new commits to your branch rather than rebasing. ## Code style Sydent follows the [Synapse code style]. [Synapse code style]: https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md Please ensure your changes match the cosmetic style of the existing project, and **never** mix cosmetic and functional changes in the same commit, as it makes it horribly hard to review otherwise. ## Changelog All changes, even minor ones, need a corresponding changelog / newsfragment entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier). To create a changelog entry, make a new file in the `changelog.d` directory named in the format of `PRnumber.type`. The type can be one of the following: * `feature` * `bugfix` * `docker` (for updates to the Docker image) * `doc` (for updates to the documentation) * `removal` (also used for deprecations) * `misc` (for internal-only changes) This file will become part of our [changelog]( https://github.com/matrix-org/sydent/blob/master/CHANGELOG.md) at the next release, so the content of the file should be a short description of your change in the same style as the rest of the changelog. The file can contain Markdown formatting, and should end with a full stop (.) or an exclamation mark (!) for consistency. **PLEASE DO** add credits for yourself to your changelog entry, by writing 'Contributed by *Your Name*.' or 'Contributed by @*your-github-username*.' at the end of your changelog entry, unless you would prefer not to. We value your contributions and would like to have you shouted out in the release notes! For example, a fix in PR #1234 would have its changelog entry in `changelog.d/1234.bugfix`, and contain content like: > The security levels of Florbs are now validated when received > via the `/federation/florb` endpoint. Contributed by Jane Matrix. If there are multiple pull requests involved in a single bugfix/feature/etc, then the content for each `changelog.d` file should be the same. Towncrier will merge the matching files together into a single changelog entry when we come to release. ### How do I know what to call the changelog file before I create the PR? Obviously, you don't know if you should call your newsfile `1234.bugfix` or `5678.bugfix` until you create the PR, which leads to a chicken-and-egg problem. There are two options for solving this: 1. Open the PR without a changelog file, see what number you got, and *then* add the changelog file to your branch (see [Updating your pull request](#updating-your-pull-request)), or: 2. Look at the [list of all issues/PRs](https://github.com/matrix-org/sydent/issues?q=), add one to the highest number you see, and quickly open the PR before somebody else claims your number. [This script](https://github.com/richvdh/scripts/blob/master/next_github_number.sh) might be helpful if you find yourself doing this a lot. Sorry, we know it's a bit fiddly, but it's *really* helpful for us when we come to put together a release! ## Sign off In order to have a concrete record that your contribution is intentional and you agree to license it under the same terms as the project's license, we've adopted the same lightweight approach that the Linux Kernel [submitting patches process]( https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other projects use: the DCO (Developer Certificate of Origin: https://developercertificate.org/). This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix: ``` Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 660 York Street, Suite 102, San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` If you agree to this for your contribution, then all that's needed is to include the line in your commit or pull request comment: ``` Signed-off-by: Your Name ``` We accept contributions under a legally identifiable name, such as your name on government documentation or common-law names (names claimed by legitimate usage or repute). Unfortunately, we cannot accept anonymous contributions at this time. Git allows you to add this signoff automatically when using the `-s` flag to `git commit`, which uses the name and email set in your `user.name` and `user.email` git configs. ## Continuous integration and testing *GitHub Actions* will automatically run a series of checks and tests against any PR which is opened against the project; if your change breaks the build, this will be shown in GitHub, with links to the build results. If your build fails, please try to fix the errors and update your branch. ## Updating your pull request If you decide to make changes to your pull request - perhaps to address issues raised in a review, or to fix problems highlighted by [continuous integration](#continuous-integration-and-testing) - just add new commits to your branch, and push to GitHub. The pull request will automatically be updated. Please **avoid** rebasing your branch, especially once the PR has been reviewed: doing so makes it very difficult for a reviewer to see what has changed since a previous review. ## Conclusion That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! sydent-2.5.1/Dockerfile000066400000000000000000000030371414516477000150330ustar00rootroot00000000000000# # Step 1: Build sydent and install dependencies # FROM docker.io/python:3.8-slim as builder # Install dev packages RUN apt-get update && apt-get install -y \ build-essential # Add user sydent RUN addgroup --system --gid 993 sydent \ && adduser --disabled-password --home /sydent --system --uid 993 --gecos sydent sydent \ && echo "sydent:$(dd if=/dev/random bs=32 count=1 | base64)" | chpasswd # Copy resources COPY --chown=sydent:sydent ["res", "/sydent/res"] COPY --chown=sydent:sydent ["scripts", "/sydent/scripts"] COPY --chown=sydent:sydent ["sydent", "/sydent/sydent"] COPY --chown=sydent:sydent ["README.rst", "setup.cfg", "setup.py", "/sydent/"] # Install dependencies USER sydent WORKDIR /sydent RUN pip install --user --upgrade pip setuptools sentry-sdk prometheus_client \ && pip install --user . \ && rm -rf /sydent/.cache \ && find /sydent -name '*.pyc' -delete # # Step 2: Reduce image size and layers # FROM docker.io/python:3.8-slim # Add user sydent and create /data directory RUN addgroup --system --gid 993 sydent \ && adduser --disabled-password --home /sydent --system --uid 993 --gecos sydent sydent \ && echo "sydent:$(dd if=/dev/random bs=32 count=1 | base64)" | chpasswd \ && mkdir /data \ && chown sydent:sydent /data # Copy sydent COPY --from=builder ["/sydent", "/sydent"] ENV SYDENT_CONF=/data/sydent.conf ENV SYDENT_PID_FILE=/data/sydent.pid ENV SYDENT_DB_PATH=/data/sydent.db WORKDIR /sydent USER sydent:sydent VOLUME ["/data"] EXPOSE 8090/tcp CMD [ "python", "-m", "sydent.sydent" ] sydent-2.5.1/LICENSE000066400000000000000000000236761414516477000140610ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS sydent-2.5.1/MANIFEST.in000066400000000000000000000001621414516477000145730ustar00rootroot00000000000000include LICENSE graft res include scripts/generate-key include scripts/sydent-bind recursive-include sydent *.sql sydent-2.5.1/README.rst000066400000000000000000000137321414516477000145330ustar00rootroot00000000000000Installation ============ Installing the system dependencies ---------------------------------- To install Sydent's dependencies on a Debian-based system, run:: sudo apt-get install build-essential python3-dev libffi-dev \ sqlite3 libssl-dev python-virtualenv libxslt1-dev Creating the virtualenv ----------------------- To create the virtual environment in which Sydent will run:: virtualenv -p python3 ~/.sydent source ~/.sydent/bin/activate pip install --upgrade pip pip install --upgrade setuptools Installing the latest Sydent release from PyPI ---------------------------------------------- Sydent and its dependencies can be installed using ``pip`` by running:: pip install matrix-sydent Installing from source ---------------------- Alternatively, Sydent can be installed using ``pip`` from a local git checkout:: git clone https://github.com/matrix-org/sydent.git cd sydent pip install -e . Running Sydent ============== With the virtualenv activated, you can run Sydent using:: python -m sydent.sydent This will create a configuration file in ``sydent.conf`` with some defaults. If a setting is defined in both the ``[DEFAULT]`` section and another section in the configuration file, then the value in the other section is used. You'll most likely want to change the server name (``server.name``) and specify an email server (look for the settings starting with ``email.``). By default, Sydent will listen on ``0.0.0.0:8090``. This can be changed by changing the values for the configuration settings ``clientapi.http.bind_address`` and ``clientapi.http.port``. Sydent uses SQLite as its database backend. By default, it will create the database as ``sydent.db`` in its working directory. The name can be overridden by modifying the ``db.file`` configuration option. Sydent is known to be working with SQLite version 3.16.2 and later. SMS originators --------------- Defaults for SMS originators will not be added to the generated config file, these should be added to the ``[sms]`` section of that config file in the form:: originators. = : Where country code is the numeric country code, or ``default`` to specify the originator used for countries not listed. For example, to use a selection of long codes for the US/Canada, a short code for the UK and an alphanumertic originator for everywhere else:: originators.1 = long:12125552368,long:12125552369 originators.44 = short:12345 originators.default = alpha:Matrix Docker ====== A Dockerfile is provided for sydent. To use it, run ``docker build -t sydent .`` in a sydent checkout. To run it, use ``docker run --env=SYDENT_SERVER_NAME=my-sydent-server -p 8090:8090 sydent``. Caution: All data will be lost when the container is terminated! Persistent data --------------- By default, all data is stored in ``/data``. The best method is to put the data in a Docker volume. .. code-block:: shell docker volume create sydent-data docker run ... --mount type=volume,source=sydent-data,destination=/data sydent But you can also bind a local directory to the container. However, you then have to pay attention to the file permissions. .. code-block:: shell mkdir /path/to/sydent-data chown 993:993 /path/to/sydent-data docker run ... --mount type=bind,source=/path/to/sydent-data,destination=/data sydent Environment variables --------------------- .. warning:: These variables are only taken into account at first start and are written to the configuration file. +--------------------+-----------------+-----------------------+ | Variable Name | Sydent default | Dockerfile default | +====================+=================+=======================+ | SYDENT_SERVER_NAME | *empty* | *empty* | +--------------------+-----------------+-----------------------+ | SYDENT_CONF | ``sydent.conf`` | ``/data/sydent.conf`` | +--------------------+-----------------+-----------------------+ | SYDENT_PID_FILE | ``sydent.pid`` | ``/data/sydent.pid`` | +--------------------+-----------------+-----------------------+ | SYDENT_DB_PATH | ``sydent.db`` | ``/data/sydent.db`` | +--------------------+-----------------+-----------------------+ Internal bind and unbind API ============================ It is possible to enable an internal API which allows for binding and unbinding between identifiers and matrix IDs without any validation. This is open to abuse, so is disabled by default, and when it is enabled, is available only on a separate socket which is bound to ``localhost`` by default. To enable it, configure the port in the config file. For example:: [http] internalapi.http.port = 8091 To change the address to which that API is bound, set the ``internalapi.http.bind_address`` configuration setting in the ``[http]`` section, for example:: [http] internalapi.http.port = 8091 internalapi.http.bind_address = 192.168.0.18 As already mentioned above, this is open to abuse, so make sure this address is not publicly accessible. To use bind:: curl -XPOST 'http://localhost:8091/_matrix/identity/internal/bind' -H "Content-Type: application/json" -d '{"address": "matthew@arasphere.net", "medium": "email", "mxid": "@matthew:matrix.org"}' The response has the same format as `/_matrix/identity/api/v1/3pid/bind `_. To use unbind:: curl -XPOST 'http://localhost:8091/_matrix/identity/internal/unbind' -H "Content-Type: application/json" -d '{"address": "matthew@arasphere.net", "medium": "email", "mxid": "@matthew:matrix.org"}' The response has the same format as `/_matrix/identity/api/v1/3pid/unbind `_. Replication =========== It is possible to configure a mesh of Sydent instances which replicate identity bindings between each other. See ``_. sydent-2.5.1/changelog.d/000077500000000000000000000000001414516477000152075ustar00rootroot00000000000000sydent-2.5.1/changelog.d/.gitignore000066400000000000000000000000141414516477000171720ustar00rootroot00000000000000!.gitignore sydent-2.5.1/docs/000077500000000000000000000000001414516477000137665ustar00rootroot00000000000000sydent-2.5.1/docs/casefold_migration.md000066400000000000000000000055331414516477000201470ustar00rootroot00000000000000# Migrating to case-insensitive email addresses **Note: the operation described in this documentation is only needed if your server was running a version of Sydent earlier than 2.4.0 at some point, and only needs to be run once. If the first version of Sydent you have set up is 2.4.0 or later, or if you have already run this operation, you don't need to do it again.** In the past, the Matrix specification would consider email addresses as case-sensitive. This means `alice@example.com` and `Alice@example.com` would be seen as two different email addresses which could each be associated with a different Matrix user ID. With [MSC2265](https://github.com/matrix-org/matrix-doc/pull/2265), the Matrix specification was updated so that email addresses are considered without any case sensitivity (so the two addresses mentioned in the previous paragraph would be considered as being one and the same). As of version 2.4.0, Sydent supports this change by processing each new association without case sensitivity. However, some data might remain in the database from earlier versions when Sydent would support multiple associations for a given email address (by using variations of the same address with a different case). This means some addresses in your identity server's database might not have been stored in a format that allows for case-insensitive processing, or might have duplicate associations. To correct this, Sydent 2.4.0 introduces a [script](https://github.com/matrix-org/sydent/blob/main/scripts/casefold_db.py) that inspects an identity server's database and fixes it to be compatible with this change: ``` Usage: /path/to/sydent/scripts/casefold_db.py [--no-email] [--dry-run] /path/to/sydent.conf Arguments: * --no-email: don't send out emails when deleting associations due to duplicates * --dry-run: don't update database rows and don't send out emails ``` If the script finds a duplicate (i.e. an email address with multiple associations), it keeps the most recent association and deletes the others. If one or more of the Matrix user IDs that are being dissociated don't match the one being kept, the script also sends an email to the address to inform the user of the dissocation. The default template for this email can be found [here](https://github.com/matrix-org/sydent/blob/main/res/matrix-org/migration_template.eml.j2) and can be overriden by configuring a custom template directory (by changing the `templates.path` configuration setting). The custom template must be named `migration_template.eml.j2` (or `migration_template.eml` if not using Jinja 2 syntax), and will be given the Matrix user ID being dissociated at render through the variable `mxid`. This script is safe to run whilst Sydent is running. If the script is not run, there may be associations in your database that can no longer be looked up and duplicate associations may be registered. sydent-2.5.1/docs/replication.md000066400000000000000000000017121414516477000166220ustar00rootroot00000000000000Intra-sydent replication ------------------------ Replication takes place over HTTPs connections, using server and client TLS certificates (currently each sydent instance can only be configured with a single certificate which is used as both a server and a client certificate). Replication peers are (currently) configured in the sqlite database; you need to add a row to both the `peers` and `peer_pubkeys` tables. The `name` / `peername` in these tables must match the `server_name` in the configuration of the peer, which is the name that peer will use to sign associations. Inbound replication connections are authenticated according to the Common Name in the client certificate, so that must also match the `server_name`. By default, that name is also used for outbound connections, but it is possible to override this by adding a setting to the config file such as: [peer.example.com] base_replication_url = https://internal-address.example.com:4434 sydent-2.5.1/docs/templates.md000066400000000000000000000165411414516477000163150ustar00rootroot00000000000000# Templates Sydent uses parametrised templates to generate the content of emails it sends and webpages it shows to users. Example templates can be found in the [res](https://github.com/matrix-org/sydent/tree/main/res) folder. ## Branding Sending a value for `brand` to some API endpoints allows for different email and http templates to be used. These templates should be stored in a file structure like this: ``` root_template_dir/ brand1/ [ brand1 template files ] brand2/ [ brand2 template files ] ``` The config option `templates.root_directory` should be set to the path of `root_template_dir` and `templates.default_brand` should be set to the sub directory to use if no `brand` (or an invalid `brand`) is provided by the request. Here are the requests that can contain a value for `brand` and what template the brand value is used to select: Invite email templates: - ` POST /_matrix/identity/v1/store-invite` - ` POST /_matrix/identity/v2/store-invite` Verification email templates: - `POST /_matrix/identity/v1/validate/email/requestToken` - `POST /_matrix/identity/v2/validate/email/requestToken` Verification SMS templates - BRAND CURRENTLY IGNORED: - `POST /_matrix/identity/v1/validate/msisdn/requestToken` - `POST /_matrix/identity/v2/validate/msisdn/requestToken` Verification response templates: - `GET /_matrix/identity/v1/validate/email/submitToken` - `GET /_matrix/identity/v2/validate/email/submitToken` - `GET /_matrix/identity/v1/validate/msisdn/submitToken` - `GET /_matrix/identity/v2/validate/msisdn/submitToken` ## Template formats ### `.eml.j2` files Files ending in ".eml.j2" are Jinja templates. Using the `urlencode` Jinja filter encodes the contents for URLs suitably. All variables are automatically made safe for HTML. If needed the `safe` filter can be used to prevent the HTML encoding, this is useful for email headers and for including a plaintext portion of an email. See the Jinja [documentation](https://jinja.palletsprojects.com/en/3.0.x/templates/#variables) for more instructions on how to write these templates. ### DEPRECATED `.eml` template files Files ending in ".eml" can use python `%(variable)s` string substitution. Appending "_forurl" or "_forhtml" to any of the variable names listed below returns their values encoded suitably for URLs or HTML respectively. For example ">" in the `_forhtml` version would be replaced with ">". Note: "&&" must be used to get a raw `&` character. --- ## Invite email templates Invitation emails are sent when someone is invited to a room by their email address. They should contain a link for the user to click on that takes them to a matrix client. Invite template files should have the name `invite_template.eml.j2`. ### Substitutions from Sydent Variable | Contents ----------- | -------- `date` | The time and date of sending as defined in RFC 2822 (e.g. "Fri, 09 Nov 2001 01:08:47 -0000") `ephemeral_private_key` | The ephemeral private key being used for this invite `from` | The sending email address as configured in `email.from` `messageid` | The unique ID for this email `multipart_boundary` | Randomized multipart boundary to use in multipart emails. **NOTE: has no `_forurl` or `_forhtml` variants** `subject_header_value` | The invite subject line. As configured in `email.invite.subject` (for a room invite) and `email.invite. subject_space` (for a space invite) `to` | The destination email address `token` | A randomly generated token `web_client_location` | Default web client to invite user to join via ### Substitutions from homeservers All values that get sent to the `/_matrix/identity/v2/store-invite` API endpoint are accessible as variables in invite templates. Invites sent from Synapse make the following available: Variable | Contents ----------- | -------- `address` | The email address being invited (same as `to`) `bracketed_room_name` | The string stored in `room_name` with brackets around it `bracketed_verified_sender` | The string stored in `sender` with brackets around it `medium` | The string "email" `org.matrix.msc3288.room_type` | **UNSTABLE.** The same as `room_type` `org.matrix.web_client_location` | **UNSTABLE.** The same as `web_client_location` `room_alias` | An alias for the room (probably more readable than `room_id`) `room_avatar_url` | The URL of the room's avatar `room_id` | The ID of the room to which the user is invited `room_join_rules` | The join rules of the email (e.g. "public" or "invite") `room_name` | The m.room.name state of the room. (e.g. "Synape Admins") `room_type` | Is set to "m.space" if the invite is to a space `sender` | The user ID of the inviter `sender_avatar_url` | The URL of the inviter's avatar `sender_display_name` | The current display name of the inviter Version when table last updated: Synapse v1.42.0 ## Verification email templates Verification template files should have the name `verification_template.eml.j2`. ### All substitutions Variable | Contents ----------- | -------- `date` | The time and date of sending as defined in RFC 2822 (e.g. "Fri, 09 Nov 2001 01:08:47 -0000") `from` | The sending email address as configured in `email.from` `ipaddress` | The IP address that the verification request came from. If the IP address is unknown then this is the string "an unknown location" `messageid` | The unique ID for this email `multipart_boundary` | Randomized multipart boundary to use in multipart emails. **NOTE: has no `_forurl` or `_forhtml` variants** `to` | The destination email address `token` | A randomly generated token that some clients might need ## Verification SMS templates The SMS template isn't read from a file. Instead the SMS template should be placed as a string into the config option `bodyTemplate`. This string uses python `{variable}` substitution. ### All substitutions Variable | Contents ----------- | -------- `token` | A randomly generated token for the user to enter into the client ## Migration email templates Migration template files should have the name `migration_template.eml.j2`. The deprecated `.eml` template version is not supported. ### All substitutions Variable | Contents ----------- | -------- `date` | The time and date of sending as defined in RFC 2822 (e.g. "Fri, 09 Nov 2001 01:08:47 -0000") `from` | The sending email address as configured in `email.from` `messageid` | The unique ID for this email `multipart_boundary` | Randomized multipart boundary to use in multipart emails. **NOTE: has no `_forurl` or `_forhtml` variants** `mxid` | The user ID that has been disassociated from the destination email address `to` | The destination email address ## Verification response http templates Verification response templates should have the name `verify_response_template.http`. ### All substitutions Variable | Contents ----------- | -------- `message` | The verification success or failure message from the server sydent-2.5.1/matrix-sydent.service000066400000000000000000000006571414516477000172400ustar00rootroot00000000000000# Example systemd configuration file for sydent. Copy to # /etc/systemd/system/matrix-sydent.service, update the paths if necessary, # then: # # systemctl enable matrix-sydent # systemctl start matrix-sydent [Unit] Description="Matrix identity server" [Service] WorkingDirectory=/opt/sydent ExecStart=/opt/sydent/env/bin/python -m sydent.sydent User=sydent Group=nogroup Restart=always [Install] WantedBy=default.target sydent-2.5.1/matrix_is_test/000077500000000000000000000000001414516477000160745ustar00rootroot00000000000000sydent-2.5.1/matrix_is_test/__init__.py000066400000000000000000000000001414516477000201730ustar00rootroot00000000000000sydent-2.5.1/matrix_is_test/launcher.py000066400000000000000000000057151414516477000202570ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import tempfile import time from subprocess import Popen CFG_TEMPLATE = """ [http] clientapi.http.bind_address = localhost clientapi.http.port = {port} client_http_base = http://localhost:{port} federation.verifycerts = False [db] db.file = :memory: [general] server.name = test.local terms.path = {terms_path} templates.path = {testsubject_path}/res brand.default = is-test ip.whitelist = 127.0.0.1 [email] email.tlsmode = 0 email.invite.subject = %(sender_display_name)s has invited you to chat email.invite.subject_space = %(sender_display_name)s has invited you to a space email.smtphost = localhost email.from = Sydent Validation email.smtpport = 9925 email.subject = Your Validation Token """ class MatrixIsTestLauncher: def __init__(self, with_terms): self.with_terms = with_terms def launch(self): sydent_path = os.path.abspath( os.path.join( os.path.dirname(__file__), "..", ) ) testsubject_path = os.path.join( sydent_path, "matrix_is_test", ) terms_path = ( os.path.join(testsubject_path, "terms.yaml") if self.with_terms else "" ) port = 8099 if self.with_terms else 8098 self.tmpdir = tempfile.mkdtemp(prefix="sydenttest") with open(os.path.join(self.tmpdir, "sydent.conf"), "w") as cfgfp: cfgfp.write( CFG_TEMPLATE.format( testsubject_path=testsubject_path, terms_path=terms_path, port=port, ) ) newEnv = os.environ.copy() newEnv.update( { "PYTHONPATH": sydent_path, } ) stderr_fp = open(os.path.join(testsubject_path, "sydent.stderr"), "w") pybin = os.getenv("SYDENT_PYTHON", "python") self.process = Popen( args=[pybin, "-m", "sydent.sydent"], cwd=self.tmpdir, env=newEnv, stderr=stderr_fp, ) # XXX: wait for startup in a sensible way time.sleep(2) self._baseUrl = "http://localhost:%d" % (port,) def tearDown(self): print("Stopping sydent...") self.process.terminate() shutil.rmtree(self.tmpdir) def get_base_url(self): return self._baseUrl sydent-2.5.1/matrix_is_test/res/000077500000000000000000000000001414516477000166655ustar00rootroot00000000000000sydent-2.5.1/matrix_is_test/res/is-test/000077500000000000000000000000001414516477000202555ustar00rootroot00000000000000sydent-2.5.1/matrix_is_test/res/is-test/invite_template.eml000066400000000000000000000003471414516477000241510ustar00rootroot00000000000000{ "token": "%(token)s", "room_alias": "%(room_alias)s", "room_avatar_url": "%(room_avatar_url)s", "room_name": "%(room_name)s", "sender_display_name": "%(sender_display_name)s", "sender_avatar_url": "%(sender_avatar_url)s" } sydent-2.5.1/matrix_is_test/res/is-test/invite_template.eml.j2000066400000000000000000000003631414516477000244610ustar00rootroot00000000000000{ "token": "{{ token }}", "room_alias": "{{ room_alias }}", "room_avatar_url": "{{ room_avatar_url }}", "room_name": "{{ room_name }}", "sender_display_name": "{{ sender_display_name }}", "sender_avatar_url": "{{ sender_avatar_url }}" } sydent-2.5.1/matrix_is_test/res/is-test/verification_template.eml000066400000000000000000000000201414516477000253210ustar00rootroot00000000000000<<<%(token)s>>> sydent-2.5.1/matrix_is_test/res/is-test/verification_template.eml.j2000066400000000000000000000000221414516477000256350ustar00rootroot00000000000000<<<{{ token }}>>> sydent-2.5.1/matrix_is_test/res/is-test/verify_response_template.html000066400000000000000000000000531414516477000262560ustar00rootroot00000000000000matrix_is_tester:email_submit_get_response sydent-2.5.1/matrix_is_test/terms.yaml000066400000000000000000000011051414516477000201070ustar00rootroot00000000000000master_version: "someversion" docs: terms_of_service: version: "5.0" langs: en: name: "Terms of Service" url: "https://example.org/somewhere/terms-2.0-en.html" fr: name: "Conditions d'utilisation" url: "https://example.org/somewhere/terms-2.0-fr.html" privacy_policy: version: "1.2" langs: en: name: "Privacy Policy" url: "https://example.org/somewhere/privacy-1.2-en.html" fr: name: "Politique de confidentialité" url: "https://example.org/somewhere/privacy-1.2-fr.html" sydent-2.5.1/pyproject.toml000066400000000000000000000031141414516477000157510ustar00rootroot00000000000000[tool.towncrier] package = "sydent" filename = "CHANGELOG.md" directory = "changelog.d" issue_format = "[\\#{issue}](https://github.com/matrix-org/sydent/issues/{issue})" [[tool.towncrier.type]] directory = "feature" name = "Features" showcontent = true [[tool.towncrier.type]] directory = "bugfix" name = "Bugfixes" showcontent = true [[tool.towncrier.type]] directory = "docker" name = "Updates to the Docker image" showcontent = true [[tool.towncrier.type]] directory = "doc" name = "Improved Documentation" showcontent = true [[tool.towncrier.type]] directory = "removal" name = "Deprecations and Removals" showcontent = true [[tool.towncrier.type]] directory = "misc" name = "Internal Changes" showcontent = true [tool.isort] profile = "black" [tool.black] target-version = ['py36'] [tool.mypy] plugins = "mypy_zope:plugin" show_error_codes = true namespace_packages = true strict = true files = [ # Find files that pass with # find sydent tests -type d -not -name __pycache__ -exec bash -c "mypy --strict '{}' > /dev/null" \; -print "sydent" # TODO the rest of CI checks these---mypy ought to too. # "tests", # "matrix_is_test", # "scripts", # "setup.py", ] mypy_path = "stubs" [[tool.mypy.overrides]] module = [ "idna", "nacl.*", "netaddr", "prometheus_client", "sentry_sdk", "signedjson.*", "sortedcontainers", ] ignore_missing_imports = true sydent-2.5.1/res/000077500000000000000000000000001414516477000136275ustar00rootroot00000000000000sydent-2.5.1/res/matrix-org/000077500000000000000000000000001414516477000157205ustar00rootroot00000000000000sydent-2.5.1/res/matrix-org/invite_template.eml000066400000000000000000000117231414516477000216140ustar00rootroot00000000000000Date: %(date)s From: %(from)s To: %(to)s Message-ID: %(messageid)s Subject: %(subject_header_value)s MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="%(multipart_boundary)s" --%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, %(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room %(bracketed_room_name)son Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) %(web_client_location)s/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s&room_type=%(room_type_forurl)s About Matrix: Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history. Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. Thanks, Matrix --%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hi,

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a room %(bracketed_room_name_forhtml)s on Matrix. To join the conversation, either pick a Matrix client or use the single-click link below to join via Element (requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.)

Join the conversation.


About Matrix:

Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history.

Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones.

Thanks,

Matrix
--%(multipart_boundary)s-- sydent-2.5.1/res/matrix-org/invite_template.eml.j2000066400000000000000000000123661414516477000221320ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: {{ subject_header_value|safe }} MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, {{ sender_display_name|safe }} {{ bracketed_verified_sender|safe }}has invited you into a {% if room_type == "m.space" %}space{% else %}room{% endif %} {{ bracketed_room_name|safe }}on Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) {{ web_client_location }}/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }}&room_type={{ room_type|urlencode }} About Matrix: Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history. Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. Thanks, Matrix --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hi,

{{ sender_display_name }} {{ bracketed_verified_sender }} has invited you into a {% if room_type == "m.space" %}space{% else %}room{% endif %} {{ bracketed_room_name }} on Matrix. To join the conversation, either pick a Matrix client or use the single-click link below to join via Element (requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.)

Join the conversation.


About Matrix:

Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history.

Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones.

Thanks,

Matrix
--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/matrix-org/migration_template.eml.j2000066400000000000000000000110501414516477000226120ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: We have changed the way your Matrix account and email address are associated MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello, We’ve recently improved how people discover your Matrix account. In the past, identity services took capitalisation into account when storing email addresses. This means Alice@example.com and alice@example.com would be considered to be two different addresses, and could be associated with different Matrix accounts. We’ve now updated this behaviour so anyone can find you, no matter how your email is capitalised. As part of this recent update, we've dissociated the Matrix account {{ mxid|safe }} from this e-mail address. No action is needed on your part. This doesn’t affect any passwords or password reset options on your account. About Matrix: Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history. Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. Thanks, Matrix --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello,

We’ve recently improved how people discover your Matrix account.
In the past, identity services took capitalisation into account when storing email addresses. This means Alice@example.com and alice@example.com would be considered to be two different addresses, and could be associated with different Matrix accounts. We’ve now updated this behaviour so anyone can find you, no matter how your email is capitalised. As part of this recent update, we've dissociated the Matrix account {{ mxid|safe }} from this e-mail address.
No action is needed on your part. This doesn’t affect any passwords or password reset options on your account.


About Matrix:

Matrix.org is an open standard for interoperable, decentralised, real-time communication over IP, supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history.

Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones.

Thanks,

Matrix
--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/matrix-org/verification_template.eml000066400000000000000000000052601414516477000227770ustar00rootroot00000000000000Date: %(date)s From: %(from)s To: %(to)s Message-ID: %(messageid)s Subject: Confirm your email address for Matrix MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="%(multipart_boundary)s" --%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello, We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: %(link)s If your client requires a code, the code is %(token)s If you aren't aware of making such a request, please disregard this email. About Matrix: Matrix is an open standard for interoperable, decentralised, real-time communication over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history. Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. --%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello,

We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

Complete email verification

...or copy this link into your web browser:

%(link)s

If your client requires a code, the code is %(token)s

If you aren't aware of making such a request, please disregard this email.


About Matrix:

Matrix is an open standard for interoperable, decentralised, real-time communication over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history.

Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones.

--%(multipart_boundary)s-- sydent-2.5.1/res/matrix-org/verification_template.eml.j2000066400000000000000000000053741414516477000233170ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: Confirm your email address for Matrix MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello, We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: {{ link|safe }} If your client requires a code, the code is {{ token|safe }} If you aren't aware of making such a request, please disregard this email. About Matrix: Matrix is an open standard for interoperable, decentralised, real-time communication over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history. Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello,

We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

Complete email verification

...or copy this link into your web browser:

{{ link }}

If your client requires a code, the code is {{ token }}

If you aren't aware of making such a request, please disregard this email.


About Matrix:

Matrix is an open standard for interoperable, decentralised, real-time communication over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere you need a standard HTTP API for publishing and subscribing to data whilst tracking the conversation history.

Matrix defines the standard, and provides open source reference implementations of Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones.

--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/matrix-org/verify_response_template.html000066400000000000000000000001711414516477000237220ustar00rootroot00000000000000

%(message)s

sydent-2.5.1/res/vector-im/000077500000000000000000000000001414516477000155345ustar00rootroot00000000000000sydent-2.5.1/res/vector-im/invite_template.eml000066400000000000000000000142371414516477000214330ustar00rootroot00000000000000Date: %(date)s From: %(from)s To: %(to)s Message-ID: %(messageid)s Subject: %(subject_header_value)s MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="%(multipart_boundary)s" --%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, %(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room %(bracketed_room_name)son Element. To join the conversation please follow the link below. %(web_client_location)s/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s&room_type=%(room_type_forurl)s Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. Thanks, Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hi,

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a room %(bracketed_room_name_forhtml)s on Element.

Join the conversation.

Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more.

Please note that Element requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.

Thanks,

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--%(multipart_boundary)s-- sydent-2.5.1/res/vector-im/invite_template.eml.j2000066400000000000000000000146771414516477000217550ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: {{ subject_header_value|safe }} MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, {{ sender_display_name|safe }} {{ bracketed_verified_sender|safe }}has invited you into a {% if room_type == "m.space" %}space{% else %}room{% endif %} {{ bracketed_room_name|safe }}on Element. To join the conversation please follow the link below. {{ web_client_location }}/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }}&room_type={{ room_type|urlencode }} Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more. Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. Thanks, Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hi,

{{ sender_display_name }} {{ bracketed_verified_sender }} has invited you into a {% if room_type == "m.space" %}space{% else %}room{% endif %} {{ bracketed_room_name }} on Element.

Join the conversation.

Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges to other communication systems and much more.

Please note that Element requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.

Thanks,

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/vector-im/migration_template.eml.j2000066400000000000000000000126621414516477000224400ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: We have changed the way your Matrix account and email address are associated MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello, We’ve recently improved how people discover your Matrix account. In the past, identity services took capitalisation into account when storing email addresses. This means Alice@example.com and alice@example.com would be considered to be two different addresses, and could be associated with different Matrix accounts. We’ve now updated this behaviour so anyone can find you, no matter how your email is capitalised. As part of this recent update, we've dissociated the Matrix account {{ mxid|safe }} from this e-mail address. No action is needed on your part. This doesn’t affect any passwords or password reset options on your account. Thanks, Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello,

We’ve recently improved how people discover your Matrix account.
In the past, identity services took capitalisation into account when storing email addresses. This means Alice@example.com and alice@example.com would be considered to be two different addresses, and could be associated with different Matrix accounts. We’ve now updated this behaviour so anyone can find you, no matter how your email is capitalised. As part of this recent update, we've dissociated the Matrix account {{ mxid|safe }} from this e-mail address.
No action is needed on your part. This doesn’t affect any passwords or password reset options on your account.

Thanks,

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/vector-im/verification_template.eml000066400000000000000000000133411414516477000226120ustar00rootroot00000000000000Date: %(date)s From: %(from)s To: %(to)s Message-ID: %(messageid)s Subject: Confirm your email address for Element MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="%(multipart_boundary)s" --%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello there! You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: %(link)s Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. If you didn't make this request, you can safely disregard this email. Thanks! Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello there!

You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix.

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

Complete email verification

Please note that Element requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.

If you didn't make this request, you can safely disregard this email.

Thanks!

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--%(multipart_boundary)s-- sydent-2.5.1/res/vector-im/verification_template.eml.j2000066400000000000000000000134431414516477000231270ustar00rootroot00000000000000Date: {{ date|safe }} From: {{ from|safe }} To: {{ to|safe }} Message-ID: {{ messageid|safe }} Subject: Confirm your email address for Element MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hello there! You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: {{ link|safe }} Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. If you didn't make this request, you can safely disregard this email. Thanks! Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello there!

You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix.

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

Complete email verification

Please note that Element requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.

If you didn't make this request, you can safely disregard this email.

Thanks!

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--{{ multipart_boundary|safe }}-- sydent-2.5.1/res/vector-im/verify_response_template.html000066400000000000000000000017231414516477000235420ustar00rootroot00000000000000

%(message)s

sydent-2.5.1/res/vector_verification_sample.txt000066400000000000000000000127431414516477000220040ustar00rootroot00000000000000Hello there! You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: https://link_test.com Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. If you didn't make this request, you can safely disregard this email. Thanks! Element About Element: Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges. Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet. Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation. Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. --aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Content-Type: text/html; charset=UTF-8 Content-Disposition: inline

Hello there!

You have asked us to register this email address with element.io - the open source, distributed and secure shared workspace for the web that's built on Matrix.

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

Complete email verification

Please note that Element requires Chrome, Firefox or Safari on the web, or iOS or Android on mobile.

If you didn't make this request, you can safely disregard this email.

Thanks!

Element


About Element:

Break through - Element allows teams to communicate across a wide range of collaboration apps. If some team members use Element while others use IRC, Slack or Gitter, Element will allow these team members to seamlessly work together. Element offers the richest network of communication bridges.

Own Your Own Data - No one should control your communication and data but you. Element lets you run your own server, and provides users and teams with the most advanced crypto ratchet technology available today for a decentralized secure Internet.

Open Source - Element is entirely open source: all the code is published on GitHub (Apache License) for anyone to see and extend. This means teams can customize or contribute to the code and everyone can benefit from the speed of community innovation.

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org.
--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--sydent-2.5.1/scripts-dev/000077500000000000000000000000001414516477000153015ustar00rootroot00000000000000sydent-2.5.1/scripts-dev/check_newsfragment.sh000077500000000000000000000035571414516477000215070ustar00rootroot00000000000000#!/usr/bin/env bash # # A script which checks that an appropriate news file has been added on this # branch. # # As first argument, it requires the PR number, so that it can check that the # newsfragment has the correct name. # # Usage: # ./scripts-dev/check_newsfragment.sh 382 # # Exit codes: # 0: all is well # 1: the newsfragment is wrong in some way # 9: the script has not been invoked properly echo -e "+++ \e[32mChecking newsfragment\e[m" set -e if [ -z "$1" ]; then echo "Please specify the PR number as the first argument (e.g. 382)." exit 9 fi pull_request_number="$1" # Print a link to the contributing guide if the user makes a mistake CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing your changelog entry: https://github.com/matrix-org/sydent/blob/main/CONTRIBUTING.md#changelog" # If towncrier returns a non-zero exit code, print the contributing guide link and exit python -m towncrier.check --compare-with="origin/main" || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1) echo echo "--------------------------" echo matched=0 for f in `git diff --name-only origin/main... -- changelog.d`; do # check that any modified newsfiles on this branch end with a full stop. lastchar=`tr -d '\n' < $f | tail -c 1` if [ $lastchar != '.' -a $lastchar != '!' ]; then echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2 echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 exit 1 fi # see if this newsfile corresponds to the right PR [[ -n "$pull_request_number" && "$f" == changelog.d/"$pull_request_number".* ]] && matched=1 done if [[ -n "$pull_request_number" && "$matched" -eq 0 ]]; then echo -e "\e[31mERROR: Did not find a news fragment with the right number: expected changelog.d/$pull_request_number.*.\e[39m" >&2 echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 exit 1 fi sydent-2.5.1/scripts-dev/lint.sh000077500000000000000000000012461414516477000166110ustar00rootroot00000000000000#! /usr/bin/env bash set -ex # Keep this up to date with the CI config at .github/workflows/pipeline.yml black sydent/ stubs/ tests/ matrix_is_test/ scripts/ setup.py flake8 sydent/ tests/ matrix_is_test/ scripts/ setup.py # There's a different convention for formatting stub files. # Ignore various error codes from pycodestyle that we don't want # to enforce for stubs. (We rely on `black` to format stubs.) # E301, E302 and E305 complain about missing blank lines. # E701 and E7044 complains when we define a function or class entirely within # one line. flake8 stubs/ --ignore E301,E302,E305,E701,E704 isort sydent/ stubs/ tests/ matrix_is_test/ scripts/ setup.py mypy sydent-2.5.1/scripts/000077500000000000000000000000001414516477000145255ustar00rootroot00000000000000sydent-2.5.1/scripts/casefold_db.py000077500000000000000000000342041414516477000173320ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import json import logging import os import sqlite3 import sys import time from typing import Any, Dict, List, Optional, Tuple import attr import signedjson.sign from sydent.config import SydentConfig from sydent.sydent import Sydent from sydent.util import json_decoder from sydent.util.emailutils import EmailSendException, sendEmail from sydent.util.hash import sha256_and_url_safe_base64 from tests.utils import ResolvingMemoryReactorClock logger = logging.getLogger("casefold_db") # Maximum number of attempts to send an email. MAX_ATTEMPTS_FOR_EMAIL = 5 @attr.s(auto_attribs=True) class UpdateDelta: """A row to update in the local_threepid_associations table.""" address: str mxid: str lookup_hash: str @attr.s(auto_attribs=True) class DeleteDelta: """A row to delete from the local_threepid_associations table.""" address: str mxid: str @attr.s(auto_attribs=True) class Delta: """Delta to apply to the local_threepid_associations table for a single case-insensitive email address. """ to_update: UpdateDelta to_delete: Optional[List[DeleteDelta]] = None class CantSendEmailException(Exception): """Raised when we didn't succeed to send an email after MAX_ATTEMPTS_FOR_EMAIL attempts. """ pass def calculate_lookup_hash(sydent, address): cur = sydent.db.cursor() pepper_result = cur.execute("SELECT lookup_pepper from hashing_metadata") pepper = pepper_result.fetchone()[0] combo = "%s %s %s" % (address, "email", pepper) lookup_hash = sha256_and_url_safe_base64(combo) return lookup_hash def sendEmailWithBackoff( sydent: Sydent, address: str, mxid: str, test: bool = False, ) -> None: """Send an email with exponential backoff - that way we don't stop sending halfway through if the SMTP server rejects our email (e.g. because of rate limiting). Setting test to True disables the backoff. Raises a CantSendEmailException if no email could be sent after MAX_ATTEMPTS_FOR_EMAIL attempts. """ # Disable backoff if we're running tests. backoff = 1 if not test else 0 for i in range(MAX_ATTEMPTS_FOR_EMAIL): try: template_file = sydent.get_branded_template( None, "migration_template.eml", ) sendEmail( sydent, template_file, address, {"mxid": mxid}, log_send_errors=False, ) logger.info("Sent email to %s" % address) return except EmailSendException: logger.info( "Failed to send email to %s (attempt %d/%d)" % (address, i + 1, MAX_ATTEMPTS_FOR_EMAIL) ) time.sleep(backoff) backoff *= 2 raise CantSendEmailException() def update_local_associations( sydent: Sydent, db: sqlite3.Connection, send_email: bool, dry_run: bool, test: bool = False, ) -> None: """Update the DB table local_threepid_associations so that all stored emails are casefolded, and any duplicate mxid's associated with the given email are deleted. Setting dry_run to True means that the script is being run in dry-run mode by the user, i.e. it will run but will not send any email nor update the database. Setting test to True means that the function is being called as part of an automated test, and therefore we shouldn't backoff when sending emails. :return: None """ logger.info("Processing rows in local_threepid_associations") res = db.execute( "SELECT address, mxid FROM local_threepid_associations WHERE medium = 'email'" "ORDER BY ts DESC" ) # a dict that associates an email address with correspoinding mxids and lookup hashes associations: Dict[str, List[Tuple[str, str, str]]] = {} logger.info("Computing new hashes and signatures for local_threepid_associations") # iterate through selected associations, casefold email, rehash it, and add to # associations dict for address, mxid in res.fetchall(): casefold_address = address.casefold() # rehash email since hashes are case-sensitive lookup_hash = calculate_lookup_hash(sydent, casefold_address) if casefold_address in associations: associations[casefold_address].append((address, mxid, lookup_hash)) else: associations[casefold_address] = [(address, mxid, lookup_hash)] # Deltas to apply to the database, associated with the casefolded address they're for. deltas: Dict[str, Delta] = {} # Iterate through the results, to build the deltas. for casefold_address, assoc_tuples in associations.items(): # If the row is already in the right state and there's no duplicate, don't compute # a delta for it. if len(assoc_tuples) == 1 and assoc_tuples[0][0] == casefold_address: continue deltas[casefold_address] = Delta( to_update=UpdateDelta( address=assoc_tuples[0][0], mxid=assoc_tuples[0][1], lookup_hash=assoc_tuples[0][2], ) ) if len(assoc_tuples) > 1: # Iterate over all associations except for the first one, since we've already # processed it. deltas[casefold_address].to_delete = [] for address, mxid, _ in assoc_tuples[1:]: deltas[casefold_address].to_delete.append( DeleteDelta( address=address, mxid=mxid, ) ) logger.info(f"{len(deltas)} rows to update in local_threepid_associations") # Apply the deltas for casefolded_address, delta in deltas.items(): if not test: log_msg = f"Updating {casefolded_address}" if delta.to_delete is not None: log_msg += ( f" and deleting {len(delta.to_delete)} rows associated with it" ) logger.info(log_msg) try: # Delete each association, and send an email mentioning the affected MXID. if delta.to_delete is not None: for to_delete in delta.to_delete: if send_email and not dry_run: # If the MXID is one that will still be associated with this # email address after this run, don't send an email for it. if to_delete.mxid == delta.to_update.mxid: continue sendEmailWithBackoff( sydent, to_delete.address, to_delete.mxid, test=test, ) if not dry_run: cur = db.cursor() cur.execute( "DELETE FROM local_threepid_associations WHERE medium = 'email' AND address = ?", (to_delete.address,), ) db.commit() # Update the row now that there's no duplicate. if not dry_run: cur = db.cursor() cur.execute( "UPDATE local_threepid_associations SET address = ?, lookup_hash = ? WHERE medium = 'email' AND address = ? AND mxid = ?", ( casefolded_address, delta.to_update.lookup_hash, delta.to_update.address, delta.to_update.mxid, ), ) db.commit() except CantSendEmailException: # If we failed because we couldn't send an email move on to the next address # to de-duplicate. # We catch this error here rather than when sending the email because we want # to avoid deleting rows we can't warn users about, and we don't want to # proceed with the subsequent update because there might still be duplicates # in the database (since we haven't deleted everything we wanted to delete). continue def update_global_associations( sydent: Sydent, db: sqlite3.Connection, dry_run: bool, ) -> None: """Update the DB table global_threepid_associations so that all stored emails are casefolded, the signed association is re-signed and any duplicate mxid's associated with the given email are deleted. Setting dry_run to True means that the script is being run in dry-run mode by the user, i.e. it will run but will not send any email nor update the database. :return: None """ logger.info("Processing rows in global_threepid_associations") # get every row where the local server is origin server and medium is email origin_server = sydent.config.general.server_name medium = "email" res = db.execute( "SELECT address, mxid, sgAssoc FROM global_threepid_associations WHERE medium = ?" "AND originServer = ? ORDER BY ts DESC", (medium, origin_server), ) # dict that stores email address with mxid, email address, lookup hash, and # signed association associations: Dict[str, List[Tuple[str, str, str, str]]] = {} logger.info("Computing new hashes and signatures for global_threepid_associations") # iterate through selected associations, casefold email, rehash it, re-sign the # associations and add to associations dict for address, mxid, sg_assoc in res.fetchall(): casefold_address = address.casefold() # rehash the email since hash functions are case-sensitive lookup_hash = calculate_lookup_hash(sydent, casefold_address) # update signed associations with new casefolded address and re-sign sg_assoc = json_decoder.decode(sg_assoc) sg_assoc["address"] = address.casefold() sg_assoc = json.dumps( signedjson.sign.sign_json( sg_assoc, sydent.config.general.server_name, sydent.keyring.ed25519 ) ) if casefold_address in associations: associations[casefold_address].append( (address, mxid, lookup_hash, sg_assoc) ) else: associations[casefold_address] = [(address, mxid, lookup_hash, sg_assoc)] # list of arguments to update db with db_update_args: List[Tuple[Any, str, str, str, str]] = [] # list of mxids to delete to_delete: List[Tuple[str]] = [] for casefold_address, assoc_tuples in associations.items(): # If the row is already in the right state and there's no duplicate, don't compute # a delta for it. if len(assoc_tuples) == 1 and assoc_tuples[0][0] == casefold_address: continue db_update_args.append( ( casefold_address, assoc_tuples[0][2], assoc_tuples[0][3], assoc_tuples[0][0], assoc_tuples[0][1], ) ) if len(assoc_tuples) > 1: # Iterate over all associations except for the first one, since we've already # processed it. for address, mxid, _, _ in assoc_tuples[1:]: to_delete.append((address,)) logger.info( f"{len(to_delete)} rows to delete, {len(db_update_args)} rows to update in global_threepid_associations" ) if not dry_run: cur = db.cursor() if len(to_delete) > 0: cur.executemany( "DELETE FROM global_threepid_associations WHERE medium = 'email' AND address = ?", to_delete, ) logger.info( f"{len(to_delete)} rows deleted from global_threepid_associations" ) if len(db_update_args) > 0: cur.executemany( "UPDATE global_threepid_associations SET address = ?, lookup_hash = ?, sgAssoc = ? WHERE medium = 'email' AND address = ? AND mxid = ?", db_update_args, ) logger.info( f"{len(db_update_args)} rows updated in global_threepid_associations" ) db.commit() if __name__ == "__main__": parser = argparse.ArgumentParser(description="Casefold email addresses in database") parser.add_argument( "--no-email", action="store_true", help="run script but do not send emails" ) parser.add_argument( "--dry-run", action="store_true", help="run script but do not send emails or alter database", ) parser.add_argument("config_path", help="path to the sydent configuration file") args = parser.parse_args() # Set up logging. log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s" formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.setLevel(logging.INFO) logger.addHandler(handler) # if the path the user gives us doesn't work, find it for them if not os.path.exists(args.config_path): logger.error(f"The config file '{args.config_path}' does not exist.") sys.exit(1) sydent_config = SydentConfig() sydent_config.parse_config_file(args.config_path) reactor = ResolvingMemoryReactorClock() sydent = Sydent(sydent_config, reactor, False) update_global_associations(sydent, sydent.db, dry_run=args.dry_run) update_local_associations( sydent, sydent.db, send_email=not args.no_email, dry_run=args.dry_run, ) sydent-2.5.1/scripts/generate-key000077500000000000000000000011311414516477000170270ustar00rootroot00000000000000#!/usr/bin/env python3 # Run example # ./scripts/generate-key # Use this to generate a signing key and verify key for use in sydent # configurations. # The signing key is generally used in "ed25519.signingkey" in the sydent config import sys import signedjson.key signing_key = signedjson.key.generate_signing_key(0); sk_str = "%s %s %s" % ( signing_key.alg, signing_key.version, signedjson.key.encode_signing_key_base64(signing_key) ) print ("signing key: %s " % sk_str) pk_str = signedjson.key.encode_verify_key_base64(signing_key.verify_key) print ("verify key: %s" % pk_str) sydent-2.5.1/scripts/sydent-bind000077500000000000000000000020371414516477000166750ustar00rootroot00000000000000#!/bin/bash -eu if [[ $# != 2 ]]; then echo >&2 "usage: $0 email mxid" exit 1 fi email="$1" mxid="$2" client_secret="$(uuidgen)" curl -d "client_secret=${client_secret}" -d email=${email} -d send_attempt=1 http://localhost:8090/_matrix/identity/api/v1/validate/email/requestToken sid=$(sqlite3 sydent.db "select threepid_validation_sessions.id from threepid_token_auths join threepid_validation_sessions on threepid_validation_sessions.id == threepid_token_auths.validationSession where threepid_validation_sessions.address = \"${email}\";") token=$(sqlite3 sydent.db "select token from threepid_token_auths join threepid_validation_sessions on threepid_validation_sessions.id == threepid_token_auths.validationSession where threepid_validation_sessions.address = \"${email}\";") curl "http://localhost:8090/_matrix/identity/api/v1/validate/email/submitToken?token=${token}&client_secret=${client_secret}&sid=${sid}" curl -d "sid=${sid}" -d "mxid=${mxid}" -d "client_secret=${client_secret}" http://localhost:8090/_matrix/identity/api/v1/3pid/bind sydent-2.5.1/setup.cfg000066400000000000000000000003331414516477000146560ustar00rootroot00000000000000[build_sphinx] source-dir = docs/sphinx build-dir = docs/build all_files = 1 [aliases] test = trial [trial] test_suite = tests [flake8] max-line-length = 88 extend-ignore = E203,E501,F821 # TODO: Fix E501 and F821 sydent-2.5.1/setup.py000066400000000000000000000043631414516477000145560ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd. # Copyright 2018 New Vector Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re from setuptools import find_packages, setup def read_version(): fn = os.path.join(os.path.dirname(__file__), "sydent", "__init__.py") with open(fn) as fp: f = fp.read() return re.search(r'^__version__ = "(.*)"', f).group(1) # Utility function to read the README file. # Used for the long_description. It's nice, because now 1) we have a top level # README file and 2) it's easier to type in the README file than to put a raw # string in below ... def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup( name="matrix-sydent", version=read_version(), packages=find_packages(), description="Reference Matrix Identity Verification and Lookup Server", python_requires=">=3.6", install_requires=[ "jinja2>=3.0.0", "signedjson==1.1.1", "unpaddedbase64>=1.1.0", "Twisted>=18.4.0", # twisted warns about about the absence of this "service_identity>=1.0.0", "phonenumbers>=8.12.32", "pyopenssl", "attrs>=19.1.0", "netaddr>=0.7.0", "sortedcontainers>=2.1.0", "pyyaml>=3.11", ], extras_require={ "dev": [ "parameterized==0.8.1", "flake8==3.9.2", "flake8-pyi==20.10.0", "black==21.6b0", "isort==5.8.0", "mypy>=0.902", "mypy-zope>=0.3.1", "types-Jinja2", "types-PyOpenSSL", "types-PyYAML", "types-mock", ], }, # make sure we package the sql files include_package_data=True, long_description=read("README.rst"), ) sydent-2.5.1/stubs/000077500000000000000000000000001414516477000141765ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/000077500000000000000000000000001414516477000156615ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/__init__.pyi000066400000000000000000000000001414516477000201310ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/internet/000077500000000000000000000000001414516477000175115ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/internet/__init__.pyi000066400000000000000000000000001414516477000217610ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/internet/endpoints.pyi000066400000000000000000000020401414516477000222330ustar00rootroot00000000000000from typing import AnyStr, Optional from twisted.internet import interfaces from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IProtocol, IProtocolFactory, IStreamClientEndpoint, ) from zope.interface import implementer @implementer(interfaces.IStreamClientEndpoint) class HostnameEndpoint: # Reactor should be a "provider of L{IReactorTCP}, L{IReactorTime} and # either L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}." # I don't know how to encode that in the type system. def __init__( self, reactor: object, host: AnyStr, port: int, timeout: float = ..., bindAddress: Optional[bytes] = ..., attemptDelay: Optional[float] = ..., ): ... def connect(self, protocol_factory: IProtocolFactory) -> Deferred[IProtocol]: ... def wrapClientTLS( connectionCreator: IOpenSSLClientConnectionCreator, wrappedEndpoint: IStreamClientEndpoint, ) -> IStreamClientEndpoint: ... sydent-2.5.1/stubs/twisted/internet/error.pyi000066400000000000000000000002621414516477000213650ustar00rootroot00000000000000from typing import Any, Optional class ConnectError(Exception): def __init__(self, osError: Optional[Any] = ..., string: str = ...): ... class DNSLookupError(IOError): ... sydent-2.5.1/stubs/twisted/internet/ssl.pyi000066400000000000000000000031571414516477000210430ustar00rootroot00000000000000from typing import Any, AnyStr, Dict, List, Optional, Type, TypeVar import OpenSSL.SSL # I don't like importing from _sslverify, but IOpenSSLTrustRoot isn't re-exported # anywhere else in twisted. from twisted.internet._sslverify import IOpenSSLTrustRoot, KeyPair from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IOpenSSLContextFactory, ) from zope.interface import implementer _C = TypeVar("_C") class Certificate: original: OpenSSL.crypto.X509 @classmethod def loadPEM(cls: Type[_C], data: AnyStr) -> _C: ... def platformTrust() -> IOpenSSLTrustRoot: ... class PrivateCertificate(Certificate): # PrivateKey is not set until you call _setPrivateKey, e.g. via load() privateKey: KeyPair @implementer(IOpenSSLContextFactory) class CertificateOptions: def __init__( self, trustRoot: Optional[IOpenSSLTrustRoot] = ..., **kwargs: object ): ... def _makeContext(self) -> OpenSSL.SSL.Context: ... def getContext(self) -> OpenSSL.SSL.Context: ... def optionsForClientTLS( hostname: str, trustRoot: Optional[IOpenSSLTrustRoot] = ..., clientCertificate: Optional[PrivateCertificate] = ..., acceptableProtocols: Optional[List[bytes]] = ..., *, # Shouldn't use extraCertificateOptions: # "any time you need to pass an option here that is a bug in this interface." extraCertificateOptions: Optional[Dict[Any, Any]] = ..., ) -> IOpenSSLClientConnectionCreator: ... # Type ignore: I don't want to respecify the methods on the interface that we # don't use. @implementer(IOpenSSLTrustRoot) # type: ignore[misc] class OpenSSLDefaultPaths: ... sydent-2.5.1/stubs/twisted/names/000077500000000000000000000000001414516477000167645ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/names/__init__.pyi000066400000000000000000000000001414516477000212340ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/names/dns.pyi000066400000000000000000000007271414516477000203010ustar00rootroot00000000000000from typing import ClassVar, Generic, Optional, TypeVar class Name: name: bytes def __init__(self, name: bytes = ...): ... SRV: int class Record_SRV: priority: int weight: int port: int target: Name ttl: int _Payload = TypeVar("_Payload") # should be bound to IEncodableRecord class RRHeader(Generic[_Payload]): fmt: ClassVar[str] name: Name type: int cls: int ttl: int payload: Optional[_Payload] auth: bool sydent-2.5.1/stubs/twisted/python/000077500000000000000000000000001414516477000172025ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/python/__init__.pyi000066400000000000000000000000001414516477000214520ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/python/failure.pyi000066400000000000000000000012631414516477000213560ustar00rootroot00000000000000from types import TracebackType from typing import Optional, Type, TypeVar, Union, overload _E = TypeVar("_E") class Failure(BaseException): def __init__( self, exc_value: Optional[BaseException] = ..., exc_type: Optional[Type[BaseException]] = ..., exc_tb: Optional[TracebackType] = ..., captureVars: bool = ..., ): ... @overload def check(self, singleErrorType: Type[_E]) -> Optional[_E]: ... @overload def check( self, *errorTypes: Union[str, Type[Exception]] ) -> Optional[Exception]: ... def getTraceback( self, elideFrameworkCode: int = ..., detail: str = ..., ) -> str: ... sydent-2.5.1/stubs/twisted/python/log.pyi000066400000000000000000000006101414516477000205030ustar00rootroot00000000000000from typing import Any, Dict, Optional, Union from twisted.python.failure import Failure EventDict = Dict[str, Any] def err( _stuff: Union[None, Exception, Failure] = ..., _why: Optional[str] = ..., **kw: object, ) -> None: ... class PythonLoggingObserver: def emit(self, eventDict: EventDict) -> None: ... def start(self) -> None: ... def stop(self) -> None: ... sydent-2.5.1/stubs/twisted/web/000077500000000000000000000000001414516477000164365ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/web/__init__.pyi000066400000000000000000000000001414516477000207060ustar00rootroot00000000000000sydent-2.5.1/stubs/twisted/web/client.pyi000066400000000000000000000067611414516477000204510ustar00rootroot00000000000000from typing import BinaryIO, Optional, Sequence, Type, TypeVar from twisted.internet.defer import Deferred from twisted.internet.interfaces import IConsumer, IProtocol from twisted.internet.task import Cooperator from twisted.python.failure import Failure from twisted.web.http_headers import Headers from twisted.web.iweb import ( IAgent, IAgentEndpointFactory, IBodyProducer, IPolicyForHTTPS, IResponse, ) from zope.interface import implementer _C = TypeVar("_C") class ResponseFailed(Exception): def __init__( self, reasons: Sequence[Failure], response: Optional[Response] = ... ): ... class HTTPConnectionPool: persistent: bool maxPersistentPerHost: int cachedConnectionTimeout: float retryAutomatically: bool def __init__(self, reactor: object, persistent: bool = ...): ... @implementer(IAgent) class Agent: # Here and in `usingEndpointFactory`, reactor should be a "provider of # L{IReactorTCP}, L{IReactorTime} and either # L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}." # I don't know how to encode that in the type system; see also # https://github.com/Shoobx/mypy-zope/issues/58 def __init__( self, reactor: object, contextFactory: IPolicyForHTTPS = ..., connectTimeout: Optional[float] = ..., bindAddress: Optional[bytes] = ..., pool: Optional[HTTPConnectionPool] = ..., ): ... def request( self, method: bytes, uri: bytes, headers: Optional[Headers] = ..., bodyProducer: Optional[IBodyProducer] = ..., ) -> Deferred[IResponse]: ... @classmethod def usingEndpointFactory( cls: Type[_C], reactor: object, endpointFactory: IAgentEndpointFactory, pool: Optional[HTTPConnectionPool] = ..., ) -> _C: ... @implementer(IBodyProducer) class FileBodyProducer: def __init__( self, inputFile: BinaryIO, cooperator: Cooperator = ..., readSize: int = ..., ): ... # Length is either `int` or the opaque object UNKNOWN_LENGTH. length: int | object def startProducing(self, consumer: IConsumer) -> Deferred[None]: ... def stopProducing(self) -> None: ... def pauseProducing(self) -> None: ... def resumeProducing(self) -> None: ... def readBody(response: IResponse) -> Deferred[bytes]: ... # Type ignore: I don't want to respecify the methods on the interface that we # don't use. @implementer(IResponse) # type: ignore[misc] class Response: code: int headers: Headers # Length is either `int` or the opaque object UNKNOWN_LENGTH. length: int | object def deliverBody(self, protocol: IProtocol) -> None: ... class ResponseDone: ... class URI: scheme: bytes netloc: bytes host: bytes port: int path: bytes params: bytes query: bytes fragment: bytes def __init__( self, scheme: bytes, netloc: bytes, host: bytes, port: int, path: bytes, params: bytes, query: bytes, fragment: bytes, ): ... @classmethod def fromBytes( cls: Type[_C], uri: bytes, defaultPort: Optional[int] = ... ) -> _C: ... @implementer(IAgent) class RedirectAgent: def __init__(self, agent: Agent, redirectLimit: int = ...): ... def request( self, method: bytes, uri: bytes, headers: Optional[Headers] = ..., bodyProducer: Optional[IBodyProducer] = ..., ) -> Deferred[IResponse]: ... sydent-2.5.1/stubs/twisted/web/http.pyi000066400000000000000000000043711414516477000201450ustar00rootroot00000000000000import typing from typing import AnyStr, Dict, List, Optional from twisted.internet import protocol from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress, ITCPTransport from twisted.logger import Logger from twisted.web.http_headers import Headers from twisted.web.iweb import IRequest from zope.interface import implementer class HTTPFactory(protocol.ServerFactory): ... class HTTPChannel: ... # Type ignore: I don't want to respecify the methods on the interface that we # don't use. @implementer(IRequest) # type: ignore[misc] class Request: # Instance attributes mentioned in the docstring method: bytes uri: bytes path: bytes args: Dict[bytes, List[bytes]] content: typing.BinaryIO cookies: List[bytes] requestHeaders: Headers responseHeaders: Headers notifications: List[Deferred[None]] _disconnected: bool _log: Logger # Other instance attributes set in __init__ channel: HTTPChannel client: IAddress # This was hard to derive. # - `transport` is `self.channel.transport` # - `self.channel` is set in the constructor, and looks like it's always # an `HTTPChannel`. # - `HTTPChannel` is a `LineReceiver` is a `Protocol` is a `BaseProtocol`. # - `BaseProtocol` sets `self.transport` to initially `None`. # # Note that `transport` is set to an ITransport in makeConnection, # so is almost certainly not None by the time it reaches our code. # # I've narrowed this to ITCPTransport because # - we use `self.transport.abortConnection`, which belongs to that interface # - twisted does too! in its implementation of HTTPChannel.forceAbortClient transport: Optional[ITCPTransport] def __init__(self, channel: HTTPChannel): ... def getHeader(self, key: AnyStr) -> Optional[AnyStr]: ... def handleContentChunk(self, data: bytes) -> None: ... def setResponseCode(self, code: int, message: Optional[bytes] = ...) -> None: ... def setHeader(self, k: AnyStr, v: AnyStr) -> None: ... def write(self, data: bytes) -> None: ... def finish(self) -> None: ... def getClientAddress(self) -> IAddress: ... class PotentialDataLoss(Exception): ... CACHED: object def stringToDatetime(dateString: bytes) -> int: ... sydent-2.5.1/stubs/twisted/web/iweb.pyi000066400000000000000000000065321414516477000201150ustar00rootroot00000000000000from typing import Any, AnyStr, BinaryIO, Dict, List, Mapping, Optional, Tuple from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IAddress, IConsumer, IOpenSSLClientConnectionCreator, IProtocol, IPushProducer, IStreamClientEndpoint, ) from twisted.python import urlpath from twisted.web.client import URI from twisted.web.http_headers import Headers from typing_extensions import Literal from zope.interface import Interface class IClientRequest(Interface): method: bytes absoluteURI: Optional[bytes] headers: Headers class IRequest(Interface): method: bytes uri: bytes path: bytes args: Mapping[bytes, List[bytes]] prepath: List[bytes] postpath: List[bytes] requestHeaders: Headers content: BinaryIO responseHeaders: Headers def getHeader(key: AnyStr) -> Optional[AnyStr]: ... def getCookie(key: bytes) -> Optional[bytes]: ... def getAllHeaders() -> Dict[bytes, bytes]: ... def getRequestHostname() -> bytes: ... def getHost() -> IAddress: ... def getClientAddress() -> IAddress: ... def getClientIP() -> Optional[str]: ... def getUser() -> str: ... def getPassword() -> str: ... def isSecure() -> bool: ... def getSession(sessionInterface: Any | None = ...) -> Any: ... def URLPath() -> urlpath.URLPath: ... def prePathURL() -> bytes: ... def rememberRootURL() -> None: ... def getRootURL() -> bytes: ... def finish() -> None: ... def write(data: bytes) -> None: ... def addCookie( k: AnyStr, v: AnyStr, expires: Optional[AnyStr] = ..., domain: Optional[AnyStr] = ..., path: Optional[AnyStr] = ..., max_age: Optional[AnyStr] = ..., comment: Optional[AnyStr] = ..., secure: Optional[bool] = ..., ) -> None: ... def setResponseCode(code: int, message: Optional[bytes] = ...) -> None: ... def setHeader(k: AnyStr, v: AnyStr) -> None: ... def redirect(url: AnyStr) -> None: ... # returns http.CACHED or False. http.CACHED is a string constant, but we # treat it as an opaque object, similar to UNKNOWN_LENGTH. def setLastModified(when: float) -> object | Literal[False]: ... def setETag(etag: str) -> object | Literal[False]: ... def setHost(host: bytes, port: int, ssl: bool = ...) -> None: ... class IBodyProducer(IPushProducer): # Length is either `int` or the opaque object UNKNOWN_LENGTH. length: int | object def startProducing(consumer: IConsumer) -> Deferred[None]: ... def stopProducing() -> None: ... class IResponse(Interface): version: Tuple[str, int, int] code: int phrase: str headers: Headers length: int | object request: IClientRequest previousResponse: Optional[IResponse] def deliverBody(protocol: IProtocol) -> None: ... def setPreviousResponse(response: IResponse) -> None: ... class IAgent(Interface): def request( method: bytes, uri: bytes, headers: Optional[Headers] = ..., bodyProducer: Optional[IBodyProducer] = ..., ) -> Deferred[IResponse]: ... class IPolicyForHTTPS(Interface): def creatorForNetloc( hostname: bytes, port: int ) -> IOpenSSLClientConnectionCreator: ... class IAgentEndpointFactory(Interface): def endpointForURI(uri: URI) -> IStreamClientEndpoint: ... UNKNOWN_LENGTH: object sydent-2.5.1/stubs/twisted/web/resource.pyi000066400000000000000000000005531414516477000210130ustar00rootroot00000000000000from typing import ClassVar from zope.interface import Interface, implementer class IResource(Interface): isLeaf: ClassVar[bool] def __init__() -> None: ... def putChild(path: bytes, child: IResource) -> None: ... @implementer(IResource) class Resource: isLeaf: ClassVar[bool] def putChild(self, path: bytes, child: IResource) -> None: ... sydent-2.5.1/stubs/twisted/web/server.pyi000066400000000000000000000012351414516477000204700ustar00rootroot00000000000000from typing import Callable, Optional from twisted.web import http from twisted.web.resource import IResource class Request(http.Request): ... # A requestFactory is allowed to be "[a] factory which is called with (channel) # and creates L{Request} instances.". RequestFactory = Callable[[http.HTTPChannel], Request] class Site(http.HTTPFactory): displayTracebacks: bool def __init__( self, resource: IResource, requestFactory: Optional[RequestFactory] = ..., # Args and kwargs get passed to http.HTTPFactory. But we don't use them. *args: object, **kwargs: object, ): ... NOT_DONE_YET = object # Opaque sydent-2.5.1/sydent/000077500000000000000000000000001414516477000143445ustar00rootroot00000000000000sydent-2.5.1/sydent/__init__.py000066400000000000000000000000261414516477000164530ustar00rootroot00000000000000__version__ = "2.5.1" sydent-2.5.1/sydent/config/000077500000000000000000000000001414516477000156115ustar00rootroot00000000000000sydent-2.5.1/sydent/config/__init__.py000066400000000000000000000247531414516477000177350ustar00rootroot00000000000000# Copyright 2019-2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import logging.handlers import os from configparser import DEFAULTSECT, ConfigParser from typing import Dict from sydent.config.crypto import CryptoConfig from sydent.config.database import DatabaseConfig from sydent.config.email import EmailConfig from sydent.config.general import GeneralConfig from sydent.config.http import HTTPConfig from sydent.config.sms import SMSConfig logger = logging.getLogger(__name__) CONFIG_DEFAULTS = { "general": { "server.name": os.environ.get("SYDENT_SERVER_NAME", ""), "log.path": "", "log.level": "INFO", "pidfile.path": os.environ.get("SYDENT_PID_FILE", "sydent.pid"), "terms.path": "", "address_lookup_limit": "10000", # Maximum amount of addresses in a single /lookup request # The root path to use for load templates. This should contain branded # directories. Each directory should contain the following templates: # # * invite_template.eml # * verification_template.eml # * verify_response_template.html "templates.path": "res", # The brand directory to use if no brand hint (or an invalid brand hint) # is provided by the request. "brand.default": "matrix-org", # The following can be added to your local config file to enable prometheus # support. # 'prometheus_port': '8080', # The port to serve metrics on # 'prometheus_addr': '', # The address to bind to. Empty string means bind to all. # The following can be added to your local config file to enable sentry support. # 'sentry_dsn': 'https://...' # The DSN has configured in the sentry instance project. # Whether clients and homeservers can register an association using v1 endpoints. "enable_v1_associations": "true", "delete_tokens_on_bind": "true", # Prevent outgoing requests from being sent to the following blacklisted # IP address CIDR ranges. If this option is not specified or empty then # it defaults to private IP address ranges. # # The blacklist applies to all outbound requests except replication # requests. # # (0.0.0.0 and :: are always blacklisted, whether or not they are # explicitly listed here, since they correspond to unroutable # addresses.) "ip.blacklist": "", # List of IP address CIDR ranges that should be allowed for outbound # requests. This is useful for specifying exceptions to wide-ranging # blacklisted target IP ranges. # # This whitelist overrides `ip.blacklist` and defaults to an empty # list. "ip.whitelist": "", }, "db": { "db.file": os.environ.get("SYDENT_DB_PATH", "sydent.db"), }, "http": { "clientapi.http.bind_address": "::", "clientapi.http.port": "8090", "internalapi.http.bind_address": "::1", "internalapi.http.port": "", "replication.https.certfile": "", "replication.https.cacert": "", # This should only be used for testing "replication.https.bind_address": "::", "replication.https.port": "4434", "obey_x_forwarded_for": "False", "federation.verifycerts": "True", # verify_response_template is deprecated, but still used if defined. Define # templates.path and brand.default under general instead. # # 'verify_response_template': 'res/verify_response_page_template', "client_http_base": "", }, "email": { # email.template and email.invite_template are deprecated, but still used # if defined. Define templates.path and brand.default under general instead. # # 'email.template': 'res/verification_template.eml', # 'email.invite_template': 'res/invite_template.eml', "email.from": "Sydent Validation ", "email.subject": "Your Validation Token", "email.invite.subject": "%(sender_display_name)s has invited you to chat", "email.invite.subject_space": "%(sender_display_name)s has invited you to a space", "email.smtphost": "localhost", "email.smtpport": "25", "email.smtpusername": "", "email.smtppassword": "", "email.hostname": "", "email.tlsmode": "0", # The web client location which will be used if it is not provided by # the homeserver. # # This should be the scheme and hostname only, see res/invite_template.eml # for the full URL that gets generated. "email.default_web_client_location": "https://app.element.io", # When a user is invited to a room via their email address, that invite is # displayed in the room list using an obfuscated version of the user's email # address. These config options determine how much of the email address to # obfuscate. Note that the '@' sign is always included. # # If the string is longer than a configured limit below, it is truncated to that limit # with '...' added. Otherwise: # # * If the string is longer than 5 characters, it is truncated to 3 characters + '...' # * If the string is longer than 1 character, it is truncated to 1 character + '...' # * If the string is 1 character long, it is converted to '...' # # This ensures that a full email address is never shown, even if it is extremely # short. # # The number of characters from the beginning to reveal of the email's username # portion (left of the '@' sign) "email.third_party_invite_username_obfuscate_characters": "3", # The number of characters from the beginning to reveal of the email's domain # portion (right of the '@' sign) "email.third_party_invite_domain_obfuscate_characters": "3", }, "sms": { "bodyTemplate": "Your code is {token}", "username": "", "password": "", }, "crypto": { "ed25519.signingkey": "", }, } class SydentConfig: """This is the class in charge of handling Sydent's configuration. Handling of each individual section is delegated to other classes stored in a `config_sections` list. To use this class, create a new object and then call one of `parse_config_file` or `parse_config_dict` before creating the Sydent object that uses it. """ def __init__(self) -> None: self.general = GeneralConfig() self.database = DatabaseConfig() self.crypto = CryptoConfig() self.sms = SMSConfig() self.email = EmailConfig() self.http = HTTPConfig() self.config_sections = [ self.general, self.database, self.crypto, self.sms, self.email, self.http, ] def _parse_config(self, cfg: ConfigParser) -> bool: """ Run the parse_config method on each of the objects in self.config_sections :param cfg: the configuration to be parsed :return: whether or not cfg has been altered. This method CAN return True, but it *shouldn't* as this leads to altering the config file. """ needs_saving = False for section in self.config_sections: if section.parse_config(cfg): needs_saving = True return needs_saving def parse_from_config_parser(self, cfg: ConfigParser) -> bool: """ Parse the configuration from a ConfigParser object :param cfg: the configuration to be parsed :return: whether or not cfg has been altered. This method CAN return True, but it *shouldn't* as this leads to altering the config file. """ return self._parse_config(cfg) def parse_config_file(self, config_file: str) -> None: """ Parse the given config from a filepath, populating missing items and sections. :param config_file: the file to be parsed """ # If the config file doesn't exist, prepopulate the config object # with the defaults, in the right section. # # Otherwise, we have to put the defaults in the DEFAULT section, # to ensure that they don't override anyone's settings which are # in their config file in the default section (which is likely, # because sydent used to be braindead). use_defaults = not os.path.exists(config_file) cfg = ConfigParser() for sect, entries in CONFIG_DEFAULTS.items(): cfg.add_section(sect) for k, v in entries.items(): cfg.set(DEFAULTSECT if use_defaults else sect, k, v) cfg.read(config_file) # TODO: Don't alter config file when starting Sydent so that # it can be set to read-only needs_saving = self.parse_from_config_parser(cfg) if needs_saving: fp = open(config_file, "w") cfg.write(fp) fp.close() def parse_config_dict(self, config_dict: Dict[str, Dict[str, str]]) -> None: """ Parse the given config from a dictionary, populating missing items and sections :param config_dict: the configuration dictionary to be parsed """ # Build a config dictionary from the defaults merged with the given dictionary config = copy.deepcopy(CONFIG_DEFAULTS) for section, section_dict in config_dict.items(): if section not in config: config[section] = {} for option in section_dict.keys(): config[section][option] = config_dict[section][option] # Build a ConfigParser from the merged dictionary cfg = ConfigParser() for section, section_dict in config.items(): cfg.add_section(section) for option, value in section_dict.items(): cfg.set(section, option, value) self.parse_from_config_parser(cfg) sydent-2.5.1/sydent/config/_base.py000066400000000000000000000020601414516477000172320ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod from configparser import ConfigParser class BaseConfig(ABC): @abstractmethod def parse_config(self, cfg: ConfigParser) -> bool: """ Parse the a section of the config :param cfg: the configuration to be parsed :return: whether or not cfg has been altered. This method CAN return True, but it *shouldn't* as this leads to altering the config file. """ pass sydent-2.5.1/sydent/config/crypto.py000066400000000000000000000044641414516477000175130ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from configparser import ConfigParser import nacl import signedjson.key from sydent.config._base import BaseConfig class CryptoConfig(BaseConfig): def parse_config(self, cfg: "ConfigParser") -> bool: """ Parse the crypto section of the config :param cfg: the configuration to be parsed """ signing_key_str = cfg.get("crypto", "ed25519.signingkey") signing_key_parts = signing_key_str.split(" ") save_key = False if signing_key_str == "": print( "INFO: This server does not yet have an ed25519 signing key. " "Creating one and saving it in the config file." ) self.signing_key = signedjson.key.generate_signing_key("0") save_key = True elif len(signing_key_parts) == 1: # old format key print("INFO: Updating signing key format: brace yourselves") self.signing_key = nacl.signing.SigningKey( signing_key_str, encoder=nacl.encoding.HexEncoder ) self.signing_key.version = "0" self.signing_key.alg = signedjson.key.NACL_ED25519 save_key = True else: self.signing_key = signedjson.key.decode_signing_key_base64( signing_key_parts[0], signing_key_parts[1], signing_key_parts[2] ) if save_key: signing_key_str = "%s %s %s" % ( self.signing_key.alg, self.signing_key.version, signedjson.key.encode_signing_key_base64(self.signing_key), ) cfg.set("crypto", "ed25519.signingkey", signing_key_str) return True else: return False sydent-2.5.1/sydent/config/database.py000066400000000000000000000017151414516477000177330ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from configparser import ConfigParser from sydent.config._base import BaseConfig class DatabaseConfig(BaseConfig): def parse_config(self, cfg: "ConfigParser") -> bool: """ Parse the database section of the config :param cfg: the configuration to be parsed """ self.database_path = cfg.get("db", "db.file") return False sydent-2.5.1/sydent/config/email.py000066400000000000000000000053711414516477000172600ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import socket from configparser import ConfigParser from typing import Optional from sydent.config._base import BaseConfig from sydent.config.exceptions import ConfigError from sydent.util.emailutils import EmailAddressException, check_valid_email_address class EmailConfig(BaseConfig): def parse_config(self, cfg: ConfigParser) -> bool: """ Parse the email section of the config :param cfg: the configuration to be parsed """ # These two options are deprecated self.template: Optional[str] = cfg.get("email", "email.template", fallback=None) self.invite_template = cfg.get("email", "email.invite_template", fallback=None) # This isn't used anywhere... self.validation_subject = cfg.get("email", "email.subject") self.invite_subject = cfg.get("email", "email.invite.subject", raw=True) self.invite_subject_space = cfg.get( "email", "email.invite.subject_space", raw=True ) self.smtp_server = cfg.get("email", "email.smtphost") self.smtp_port = cfg.get("email", "email.smtpport") self.smtp_username = cfg.get("email", "email.smtpusername") self.smtp_password = cfg.get("email", "email.smtppassword") self.tls_mode = cfg.get("email", "email.tlsmode") # This is the fully qualified domain name for SMTP HELO/EHLO self.host_name = cfg.get("email", "email.hostname") if self.host_name == "": self.host_name = socket.getfqdn() self.sender = cfg.get("email", "email.from") try: check_valid_email_address(self.sender, allow_description=True) except EmailAddressException as e: raise ConfigError(f"Invalid email address '{self.sender}'") from e self.default_web_client_location = cfg.get( "email", "email.default_web_client_location" ) self.username_obfuscate_characters = cfg.getint( "email", "email.third_party_invite_username_obfuscate_characters" ) self.domain_obfuscate_characters = cfg.getint( "email", "email.third_party_invite_domain_obfuscate_characters" ) return False sydent-2.5.1/sydent/config/exceptions.py000066400000000000000000000011711414516477000203440ustar00rootroot00000000000000# Copyright 2021 Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. class ConfigError(Exception): pass sydent-2.5.1/sydent/config/general.py000066400000000000000000000105011414516477000175750ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from configparser import ConfigParser from typing import List from jinja2.environment import Environment from jinja2.loaders import FileSystemLoader from sydent.config._base import BaseConfig from sydent.util.ip_range import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set class GeneralConfig(BaseConfig): def parse_config(self, cfg: "ConfigParser") -> bool: """ Parse the 'general' section of the config :param cfg: the configuration to be parsed """ self.server_name = cfg.get("general", "server.name") if self.server_name == "": self.server_name = os.uname()[1] print( "WARNING: You have not specified a server name. I have guessed that this " f"server is called '{self.server_name}'. If this is incorrect, you should " "edit 'general.server.name' in the config file." ) self.log_level = cfg.get("general", "log.level") self.log_path = cfg.get("general", "log.path") # Get the possible brands by looking at directories under the # templates.path directory. self.templates_path = cfg.get("general", "templates.path") if os.path.exists(self.templates_path): self.valid_brands = { p for p in os.listdir(self.templates_path) if os.path.isdir(os.path.join(self.templates_path, p)) } else: print( f"WARNING: The path specified by 'general.templates.path' ({self.templates_path}) " "does not exist." ) # This is a legacy code-path and assumes that verify_response_template, # email.template, and email.invite_template are defined. self.valid_brands = set() self.template_environment = Environment( loader=FileSystemLoader(cfg.get("general", "templates.path")), autoescape=True, ) self.default_brand = cfg.get("general", "brand.default") self.pidfile = cfg.get("general", "pidfile.path") self.terms_path = cfg.get("general", "terms.path") self.address_lookup_limit = cfg.getint("general", "address_lookup_limit") self.prometheus_port = cfg.getint("general", "prometheus_port", fallback=None) self.prometheus_addr = cfg.get("general", "prometheus_addr", fallback=None) self.prometheus_enabled = ( self.prometheus_port is not None and self.prometheus_addr is not None ) self.sentry_enabled = cfg.has_option("general", "sentry_dsn") self.sentry_dsn = cfg.get("general", "sentry_dsn", fallback=None) self.enable_v1_associations = parse_cfg_bool( cfg.get("general", "enable_v1_associations") ) self.delete_tokens_on_bind = parse_cfg_bool( cfg.get("general", "delete_tokens_on_bind") ) ip_blacklist = list_from_comma_sep_string(cfg.get("general", "ip.blacklist")) if not ip_blacklist: ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST ip_whitelist = list_from_comma_sep_string(cfg.get("general", "ip.whitelist")) self.ip_blacklist = generate_ip_set(ip_blacklist) self.ip_whitelist = generate_ip_set(ip_whitelist) return False def list_from_comma_sep_string(rawstr: str) -> List[str]: """ Parse the a comma seperated string into a list :param rawstr: the string to be parsed """ if rawstr == "": return [] return [x.strip() for x in rawstr.split(",")] def parse_cfg_bool(value: str) -> bool: """ Parse a string config option into a boolean This method ignores capitalisation :param value: the string to be parsed """ return value.lower() == "true" sydent-2.5.1/sydent/config/http.py000066400000000000000000000051551414516477000171500ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from configparser import ConfigParser from typing import Optional from sydent.config._base import BaseConfig class HTTPConfig(BaseConfig): def parse_config(self, cfg: "ConfigParser") -> bool: """ Parse the http section of the config :param cfg: the configuration to be parsed """ # This option is deprecated self.verify_response_template = cfg.get( "http", "verify_response_template", fallback=None ) self.client_bind_address = cfg.get("http", "clientapi.http.bind_address") self.client_port = cfg.getint("http", "clientapi.http.port") # internal port is allowed to be set to an empty string in the config internal_api_port = cfg.get("http", "internalapi.http.port") self.internal_bind_address = cfg.get( "http", "internalapi.http.bind_address", fallback="::1" ) self.internal_port: Optional[int] = None if internal_api_port != "": self.internal_port = int(internal_api_port) self.cert_file = cfg.get("http", "replication.https.certfile") self.ca_cert_file = cfg.get("http", "replication.https.cacert") self.replication_bind_address = cfg.get( "http", "replication.https.bind_address" ) self.replication_port = cfg.getint("http", "replication.https.port") self.obey_x_forwarded_for = cfg.getboolean("http", "obey_x_forwarded_for") self.verify_federation_certs = cfg.getboolean("http", "federation.verifycerts") self.server_http_url_base = cfg.get("http", "client_http_base") self.base_replication_urls = {} for section in cfg.sections(): if section.startswith("peer."): # peer name is all the characters after 'peer.' peer = section[5:] if cfg.has_option(section, "base_replication_url"): base_url = cfg.get(section, "base_replication_url") self.base_replication_urls[peer] = base_url return False sydent-2.5.1/sydent/config/sms.py000066400000000000000000000054671414516477000170010ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from configparser import ConfigParser from typing import Dict, List from sydent.config._base import BaseConfig from sydent.config.exceptions import ConfigError class SMSConfig(BaseConfig): def parse_config(self, cfg: "ConfigParser") -> bool: """ Parse the sms section of the config :param cfg: the configuration to be parsed """ self.body_template = cfg.get("sms", "bodyTemplate") # Make sure username and password are bytes otherwise we can't use them with # b64encode. self.api_username = cfg.get("sms", "username").encode("UTF-8") self.api_password = cfg.get("sms", "password").encode("UTF-8") self.originators: Dict[str, List[Dict[str, str]]] = {} self.smsRules = {} for opt in cfg.options("sms"): if opt.startswith("originators."): country = opt.split(".")[1] rawVal = cfg.get("sms", opt) rawList = [i.strip() for i in rawVal.split(",")] self.originators[country] = [] for origString in rawList: parts = origString.split(":") if len(parts) != 2: raise ConfigError( "Originators must be in form: long:, short: or alpha:, separated by commas" ) if parts[0] not in ["long", "short", "alpha"]: raise ConfigError( "Invalid originator type: valid types are long, short and alpha" ) self.originators[country].append( { "type": parts[0], "text": parts[1], } ) elif opt.startswith("smsrule."): country = opt.split(".")[1] action = cfg.get("sms", opt) if action not in ["allow", "reject"]: raise ConfigError( "Invalid SMS rule action: %s, expecting 'allow' or 'reject'" % action ) self.smsRules[country] = action return False sydent-2.5.1/sydent/db/000077500000000000000000000000001414516477000147315ustar00rootroot00000000000000sydent-2.5.1/sydent/db/__init__.py000066400000000000000000000000001414516477000170300ustar00rootroot00000000000000sydent-2.5.1/sydent/db/accounts.py000066400000000000000000000073631414516477000171330ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING, Optional, Tuple, cast from sydent.users.accounts import Account if TYPE_CHECKING: from sydent.sydent import Sydent class AccountStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getAccountByToken(self, token: str) -> Optional[Account]: """ Select the account matching the given token, if any. :param token: The token to identify the account, if any. :return: The account matching the token, or None if no account matched. """ cur = self.sydent.db.cursor() res = cur.execute( "select a.user_id, a.created_ts, a.consent_version from accounts a, tokens t " "where t.user_id = a.user_id and t.token = ?", (token,), ) row: Optional[Tuple[str, int, Optional[str]]] = res.fetchone() if row is None: return None return Account(*row) def storeAccount( self, user_id: str, creation_ts: int, consent_version: Optional[str] ) -> None: """ Stores an account for the given user ID. :param user_id: The Matrix user ID to create an account for. :param creation_ts: The timestamp in milliseconds. :param consent_version: The version of the terms of services that the user last accepted. """ cur = self.sydent.db.cursor() cur.execute( "insert or ignore into accounts (user_id, created_ts, consent_version) " "values (?, ?, ?)", (user_id, creation_ts, consent_version), ) self.sydent.db.commit() def setConsentVersion(self, user_id: str, consent_version: Optional[str]) -> None: """ Saves that the given user has agreed to all of the terms in the document of the given version. :param user_id: The Matrix ID of the user that has agreed to the terms. :param consent_version: The version of the document the user has agreed to. """ cur = self.sydent.db.cursor() cur.execute( "update accounts set consent_version = ? where user_id = ?", (consent_version, user_id), ) self.sydent.db.commit() def addToken(self, user_id: str, token: str) -> None: """ Stores the authentication token for a given user. :param user_id: The Matrix user ID to save the given token for. :param token: The token to store for that user ID. """ cur = self.sydent.db.cursor() cur.execute( "insert into tokens (user_id, token) values (?, ?)", (user_id, token), ) self.sydent.db.commit() def delToken(self, token: str) -> int: """ Deletes an authentication token from the database. :param token: The token to delete from the database. """ cur = self.sydent.db.cursor() cur.execute( "delete from tokens where token = ?", (token,), ) # Cast safety: DBAPI-2 says this is a "number"; c.f. python/typeshed#6150 deleted = cast(int, cur.rowcount) self.sydent.db.commit() return deleted sydent-2.5.1/sydent/db/hashing_metadata.py000066400000000000000000000125631414516477000205730ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py from sqlite3 import Cursor from typing import TYPE_CHECKING, Callable, List, Optional, Tuple from typing_extensions import Literal if TYPE_CHECKING: from sydent.sydent import Sydent class HashingMetadataStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def get_lookup_pepper(self) -> Optional[str]: """Return the value of the current lookup pepper from the db :return: A pepper if it exists in the database, or None if one does not exist """ cur = self.sydent.db.cursor() res = cur.execute("select lookup_pepper from hashing_metadata") # Annotation safety: lookup_pepper is marked as varchar(256) in the # schema, so could be null. I.e. `row` should strictly be # Optional[Tuple[Optional[str]]. # But I think the application code is such that either # - hashing_metadata contains no rows # - or it contains exactly one row with a nonnull lookup_pepper. row: Optional[Tuple[str]] = res.fetchone() if not row: return None pepper = row[0] # Ensure we're dealing with unicode. if isinstance(pepper, bytes): pepper = pepper.decode("UTF-8") return pepper def store_lookup_pepper( self, hashing_function: Callable[[str], str], pepper: str ) -> None: """Stores a new lookup pepper in the hashing_metadata db table and rehashes all 3PIDs :param hashing_function: A function with single input and output strings :param pepper: The pepper to store in the database """ cur = self.sydent.db.cursor() # Create or update lookup_pepper sql = ( "INSERT OR REPLACE INTO hashing_metadata (id, lookup_pepper) " "VALUES (0, ?)" ) cur.execute(sql, (pepper,)) # Hand the cursor to each rehashing function # Each function will queue some rehashing db transactions self._rehash_threepids( cur, hashing_function, pepper, "local_threepid_associations" ) self._rehash_threepids( cur, hashing_function, pepper, "global_threepid_associations" ) # Commit the queued db transactions so that adding a new pepper and hashing is atomic self.sydent.db.commit() def _rehash_threepids( self, cur: Cursor, hashing_function: Callable[[str], str], pepper: str, table: Literal["local_threepid_associations", "global_threepid_associations"], ) -> None: """Rehash 3PIDs of a given table using a given hashing_function and pepper A database cursor `cur` must be passed to this function. After this function completes, the calling function should make sure to call self`self.sydent.db.commit()` to commit the made changes to the database. :param cur: Database cursor :param hashing_function: A function with single input and output strings :param pepper: A pepper to append to the end of the 3PID (after a space) before hashing :param table: The database table to perform the rehashing on """ # Get count of all 3PID records # Medium/address combos are marked as UNIQUE in the database sql = "SELECT COUNT(*) FROM %s" % table res = cur.execute(sql) row: Tuple[int] = res.fetchone() row_count = row[0] # Iterate through each medium, address combo, hash it, # and store in the db batch_size = 500 count = 0 while count < row_count: sql = "SELECT medium, address FROM %s ORDER BY id LIMIT %s OFFSET %s" % ( table, batch_size, count, ) res = cur.execute(sql) rows: List[Tuple[str, str]] = res.fetchall() for medium, address in rows: # Skip broken db entry if not medium or not address: continue # Combine the medium, address and pepper together in the # following form: "address medium pepper" # According to MSC2134: https://github.com/matrix-org/matrix-doc/pull/2134 combo = "%s %s %s" % (address, medium, pepper) # Hash the resulting string result = hashing_function(combo) # Save the result to the DB sql = ( "UPDATE %s SET lookup_hash = ? " "WHERE medium = ? AND address = ?" % table ) # Lines up the query to be executed on commit cur.execute(sql, (result, medium, address)) count += len(rows) sydent-2.5.1/sydent/db/invite_tokens.py000066400000000000000000000134701414516477000201710ustar00rootroot00000000000000# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast if TYPE_CHECKING: from sydent.sydent import Sydent class JoinTokenStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def storeToken( self, medium: str, normalised_address: str, roomId: str, sender: str, token: str ) -> None: """ Store a new invite token and its metadata. Please note that email addresses need to be casefolded before calling this function. :param medium: The medium of the 3PID the token is associated to. :param normalised_address: The address of the 3PID the token is associated to. :param roomId: The ID of the room the 3PID is invited in. :param sender: The MXID of the user that sent the invite. :param token: The token to store. """ cur = self.sydent.db.cursor() cur.execute( "INSERT INTO invite_tokens" " ('medium', 'address', 'room_id', 'sender', 'token', 'received_ts')" " VALUES (?, ?, ?, ?, ?, ?)", (medium, normalised_address, roomId, sender, token, int(time.time())), ) self.sydent.db.commit() def getTokens(self, medium: str, address: str) -> List[Dict[str, str]]: """ Retrieves the pending invites tokens for this 3PID that haven't been delivered yet. :param medium: The medium of the 3PID to get tokens for. :param address: The address of the 3PID to get tokens for. :return: A list of dicts, each containing a pending token and its metadata for this 3PID. """ cur = self.sydent.db.cursor() res = cur.execute( "SELECT medium, address, room_id, sender, token FROM invite_tokens" " WHERE medium = ? AND address = ? AND sent_ts IS NULL", ( medium, address, ), ) rows: List[Tuple[str, str, str, str, str]] = res.fetchall() ret = [] for row in rows: medium, address, roomId, sender, token = row ret.append( { "medium": medium, "address": address, "room_id": roomId, "sender": sender, "token": token, } ) return ret def markTokensAsSent(self, medium: str, address: str) -> None: """ Updates the invite tokens associated with a given 3PID to mark them as delivered to a homeserver so they're not delivered again in the future. :param medium: The medium of the 3PID to update tokens for. :param address: The address of the 3PID to update tokens for. """ cur = self.sydent.db.cursor() cur.execute( "UPDATE invite_tokens SET sent_ts = ? WHERE medium = ? AND address = ?", ( int(time.time()), medium, address, ), ) self.sydent.db.commit() def storeEphemeralPublicKey(self, publicKey: str) -> None: """ Saves the provided ephemeral public key. :param publicKey: The key to store. """ cur = self.sydent.db.cursor() cur.execute( "INSERT INTO ephemeral_public_keys" " (public_key, persistence_ts)" " VALUES (?, ?)", (publicKey, int(time.time())), ) self.sydent.db.commit() def validateEphemeralPublicKey(self, publicKey: str) -> bool: """ Checks if an ephemeral public key is valid, and, if it is, updates its verification count. :param publicKey: The public key to validate. :return: Whether the key is valid. """ cur = self.sydent.db.cursor() cur.execute( "UPDATE ephemeral_public_keys" " SET verify_count = verify_count + 1" " WHERE public_key = ?", (publicKey,), ) self.sydent.db.commit() # Cast safety: DBAPI-2 says this is a "number"; c.f. python/typeshed#6150 rows = cast(int, cur.rowcount) return rows > 0 def getSenderForToken(self, token: str) -> Optional[str]: """ Retrieves the MXID of the user that sent the invite the provided token is for. :param token: The token to retrieve the sender of. :return: The invite's sender, or None if the token doesn't match an existing invite. """ cur = self.sydent.db.cursor() res = cur.execute("SELECT sender FROM invite_tokens WHERE token = ?", (token,)) rows: List[Tuple[str]] = res.fetchall() if rows: return rows[0][0] return None def deleteTokens(self, medium: str, address: str) -> None: """ Deletes every token for a given 3PID. :param medium: The medium of the 3PID to delete tokens for. :param address: The address of the 3PID to delete tokens for. """ cur = self.sydent.db.cursor() cur.execute( "DELETE FROM invite_tokens WHERE medium = ? AND address = ?", ( medium, address, ), ) self.sydent.db.commit() sydent-2.5.1/sydent/db/invite_tokens.sql000066400000000000000000000027271414516477000203430ustar00rootroot00000000000000/* Copyright 2015 OpenMarket Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -- Note that this SQL file is not up to date, and migrations can be found in sydent/db/sqlitedb.py CREATE TABLE IF NOT EXISTS invite_tokens ( id integer primary key, medium varchar(16) not null, address varchar(256) not null, room_id varchar(256) not null, sender varchar(256) not null, token varchar(256) not null, received_ts bigint, -- When the invite was received by us from the homeserver sent_ts bigint -- When the token was sent by us to the user ); CREATE INDEX IF NOT EXISTS invite_token_medium_address on invite_tokens(medium, address); CREATE INDEX IF NOT EXISTS invite_token_token on invite_tokens(token); CREATE TABLE IF NOT EXISTS ephemeral_public_keys( id integer primary key, public_key varchar(256) not null, verify_count bigint default 0, persistence_ts bigint ); CREATE UNIQUE INDEX IF NOT EXISTS ephemeral_public_keys_index on ephemeral_public_keys(public_key); sydent-2.5.1/sydent/db/peers.py000066400000000000000000000115251414516477000164250ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from sydent.replication.peer import RemotePeer if TYPE_CHECKING: from sydent.sydent import Sydent class PeerStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getPeerByName(self, name: str) -> Optional[RemotePeer]: """ Retrieves a remote peer using it's server name. :param name: The server name of the peer. :return: The retrieved peer. """ cur = self.sydent.db.cursor() res = cur.execute( "select p.name, p.port, p.lastSentVersion, pk.alg, pk.key from peers p, peer_pubkeys pk " "where p.name = ? and pk.peername = p.name and p.active = 1", (name,), ) # Type safety: if the query returns no rows, pubkeys will be empty # and we'll return None before using serverName. Otherwise, we'll read # at least one row and assign serverName a string value, because the # `name` column is declared `not null` in the DB. serverName: str = None # type: ignore[assignment] port: Optional[int] = None lastSentVer: Optional[int] = None pubkeys: Dict[str, str] = {} row: Tuple[str, Optional[int], Optional[int], str, str] for row in res.fetchall(): serverName = row[0] port = row[1] lastSentVer = row[2] pubkeys[row[3]] = row[4] if len(pubkeys) == 0: return None p = RemotePeer(self.sydent, serverName, port, pubkeys, lastSentVer) return p def getAllPeers(self) -> List[RemotePeer]: """ Retrieve all of the remote peers from the database. :return: A list of the remote peers this server knows about. """ cur = self.sydent.db.cursor() res = cur.execute( "select p.name, p.port, p.lastSentVersion, pk.alg, pk.key from peers p, peer_pubkeys pk " "where pk.peername = p.name and p.active = 1" ) peers = [] # Safety: we need to convince ourselves that `peername` will be not None # when passed to `RemotePeer`. # # If `res` is empty, then `pubkeys` will start empty and never be written to. # So we will never create a `RemotePeer`. That's fine. # # Otherwise we process at least one row. The first row we process will # satisfy `row[0] is not None` because `name` is nonnull in the schema. # `pubkeys` will be empty, so we skip the innermost `if` and assign peername # to be a string. There are no further assignments of `None` to `peername`; # it will be a string whenever we use it. peername: str = None # type: ignore[assignment] port = None lastSentVer = None pubkeys: Dict[str, str] = {} row: Tuple[str, Optional[int], Optional[int], str, str] for row in res.fetchall(): if row[0] != peername: if len(pubkeys) > 0: p = RemotePeer(self.sydent, peername, port, pubkeys, lastSentVer) peers.append(p) pubkeys = {} peername = row[0] port = row[1] lastSentVer = row[2] pubkeys[row[3]] = row[4] if len(pubkeys) > 0: p = RemotePeer(self.sydent, peername, port, pubkeys, lastSentVer) peers.append(p) return peers def setLastSentVersionAndPokeSucceeded( self, peerName: str, lastSentVersion: Optional[int], lastPokeSucceeded: Optional[int], ) -> None: """ Sets the ID of the last association sent to a given peer and the time of the last successful request sent to that peer. :param peerName: The server name of the peer. :param lastSentVersion: The ID of the last association sent to that peer. :param lastPokeSucceeded: The timestamp in milliseconds of the last successful request sent to that peer. """ cur = self.sydent.db.cursor() cur.execute( "update peers set lastSentVersion = ?, lastPokeSucceededAt = ? " "where name = ?", (lastSentVersion, lastPokeSucceeded, peerName), ) self.sydent.db.commit() sydent-2.5.1/sydent/db/peers.sql000066400000000000000000000022471414516477000165750ustar00rootroot00000000000000/* Copyright 2014 OpenMarket Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -- Note that this SQL file is not up to date, and migrations can be found in sydent/db/sqlitedb.py CREATE TABLE IF NOT EXISTS peers (id integer primary key, name varchar(255) not null, port integer default null, lastSentVersion integer, lastPokeSucceededAt integer, active integer not null default 0); CREATE UNIQUE INDEX IF NOT EXISTS name on peers(name); CREATE TABLE IF NOT EXISTS peer_pubkeys (id integer primary key, peername varchar(255) not null, alg varchar(16) not null, key text not null, foreign key (peername) references peers (name)); CREATE UNIQUE INDEX IF NOT EXISTS peername_alg on peer_pubkeys(peername, alg); sydent-2.5.1/sydent/db/sqlitedb.py000066400000000000000000000223151414516477000171150ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import sqlite3 from typing import TYPE_CHECKING, Tuple if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class SqliteDatabase: def __init__(self, syd: "Sydent") -> None: self.sydent = syd dbFilePath = self.sydent.config.database.database_path logger.info("Using DB file %s", dbFilePath) self.db = sqlite3.connect(dbFilePath) curVer = self._getSchemaVersion() # We always run the schema files if the version is zero: either the db is # completely empty and schema-less or it has the v0 schema, which is safe to # replay the schema files. The files in the sql directory are the v0 schema, so # a new installations will start as v0 then be upgraded to the current version. if curVer == 0: self._createSchema() self._upgradeSchema() def _createSchema(self) -> None: logger.info("Running schema files...") schemaDir = os.path.dirname(__file__) c = self.db.cursor() for f in os.listdir(schemaDir): if not f.endswith(".sql"): continue scriptPath = os.path.join(schemaDir, f) fp = open(scriptPath, "r") try: logger.info("Importing %s", scriptPath) c.executescript(fp.read()) except Exception: logger.error("Error importing %s", scriptPath) raise fp.close() c.close() self.db.commit() def _upgradeSchema(self) -> None: curVer = self._getSchemaVersion() if curVer < 1: cur = self.db.cursor() # add auto_increment to the primary key of local_threepid_associations to ensure ids are never re-used, # allow the mxid column to be null to represent the deletion of a binding # and remove not null constraints on ts, notBefore and notAfter (again for when a binding has been deleted # and these wouldn't be very meaningful) logger.info("Migrating schema from v0 to v1") cur.execute("DROP INDEX IF EXISTS medium_address") cur.execute("DROP INDEX IF EXISTS local_threepid_medium_address") cur.execute( "ALTER TABLE local_threepid_associations RENAME TO old_local_threepid_associations" ) cur.execute( "CREATE TABLE local_threepid_associations (id integer primary key autoincrement, " "medium varchar(16) not null, " "address varchar(256) not null, " "mxid varchar(256), " "ts integer, " "notBefore bigint, " "notAfter bigint)" ) cur.execute( "INSERT INTO local_threepid_associations (medium, address, mxid, ts, notBefore, notAfter) " "SELECT medium, address, mxid, ts, notBefore, notAfter FROM old_local_threepid_associations" ) cur.execute( "CREATE UNIQUE INDEX local_threepid_medium_address on local_threepid_associations(medium, address)" ) cur.execute("DROP TABLE old_local_threepid_associations") # same autoincrement for global_threepid_associations (fields stay non-nullable because we don't need # entries in this table for deletions, we can just delete the rows) cur.execute("DROP INDEX IF EXISTS global_threepid_medium_address") cur.execute("DROP INDEX IF EXISTS global_threepid_medium_lower_address") cur.execute("DROP INDEX IF EXISTS global_threepid_originServer_originId") cur.execute("DROP INDEX IF EXISTS medium_lower_address") cur.execute("DROP INDEX IF EXISTS threepid_originServer_originId") cur.execute( "ALTER TABLE global_threepid_associations RENAME TO old_global_threepid_associations" ) cur.execute( "CREATE TABLE IF NOT EXISTS global_threepid_associations " "(id integer primary key autoincrement, " "medium varchar(16) not null, " "address varchar(256) not null, " "mxid varchar(256) not null, " "ts integer not null, " "notBefore bigint not null, " "notAfter integer not null, " "originServer varchar(255) not null, " "originId integer not null, " "sgAssoc text not null)" ) cur.execute( "INSERT INTO global_threepid_associations " "(medium, address, mxid, ts, notBefore, notAfter, originServer, originId, sgAssoc) " "SELECT medium, address, mxid, ts, notBefore, notAfter, originServer, originId, sgAssoc " "FROM old_global_threepid_associations" ) cur.execute( "CREATE INDEX global_threepid_medium_address on global_threepid_associations (medium, address)" ) cur.execute( "CREATE INDEX global_threepid_medium_lower_address on " "global_threepid_associations (medium, lower(address))" ) cur.execute( "CREATE UNIQUE INDEX global_threepid_originServer_originId on " "global_threepid_associations (originServer, originId)" ) cur.execute("DROP TABLE old_global_threepid_associations") self.db.commit() logger.info("v0 -> v1 schema migration complete") self._setSchemaVersion(1) if curVer < 2: logger.info("Migrating schema from v1 to v2") cur = self.db.cursor() cur.execute( "CREATE INDEX threepid_validation_sessions_mtime ON threepid_validation_sessions(mtime)" ) self.db.commit() logger.info("v1 -> v2 schema migration complete") self._setSchemaVersion(2) if curVer < 3: cur = self.db.cursor() # Add lookup_hash columns to threepid association tables cur.execute( "ALTER TABLE local_threepid_associations " "ADD COLUMN lookup_hash VARCHAR(256)" ) cur.execute( "ALTER TABLE global_threepid_associations " "ADD COLUMN lookup_hash VARCHAR(256)" ) cur.execute( "CREATE INDEX IF NOT EXISTS lookup_hash_medium " "on global_threepid_associations " "(lookup_hash, medium)" ) # Create hashing_metadata table to store the current lookup_pepper cur.execute( "CREATE TABLE IF NOT EXISTS hashing_metadata (" "id integer primary key, " "lookup_pepper varchar(256)" ")" ) self.db.commit() logger.info("v2 -> v3 schema migration complete") self._setSchemaVersion(3) if curVer < 4: cur = self.db.cursor() cur.execute( "CREATE TABLE accounts(user_id TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, consent_version TEXT)" ) cur.execute( "CREATE TABLE tokens(token TEXT NOT NULL PRIMARY KEY, user_id TEXT NOT NULL)" ) cur.execute( "CREATE TABLE accepted_terms_urls(user_id TEXT NOT NULL, url TEXT NOT NULL)" ) cur.execute( "CREATE UNIQUE INDEX accepted_terms_urls_idx ON accepted_terms_urls (user_id, url)" ) self.db.commit() logger.info("v3 -> v4 schema migration complete") self._setSchemaVersion(4) if curVer < 5: # Fix lookup_hash index for selecting on mxid instead of medium cur = self.db.cursor() cur.execute("DROP INDEX IF EXISTS lookup_hash_medium") cur.execute( "CREATE INDEX global_threepid_lookup_hash ON global_threepid_associations(lookup_hash)" ) self.db.commit() logger.info("v4 -> v5 schema migration complete") self._setSchemaVersion(5) def _getSchemaVersion(self) -> int: cur = self.db.cursor() cur.execute("PRAGMA user_version") row: Tuple[int] = cur.fetchone() return row[0] def _setSchemaVersion(self, ver: int) -> None: cur = self.db.cursor() # NB. pragma doesn't support variable substitution so we # do it in python (as a decimal so we don't risk SQL injection) cur.execute("PRAGMA user_version = %d" % (ver,)) sydent-2.5.1/sydent/db/terms.py000066400000000000000000000037151414516477000164430ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING, List if TYPE_CHECKING: from sydent.sydent import Sydent class TermsStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getAgreedUrls(self, user_id: str) -> List[str]: """ Retrieves the URLs of the terms the given user has agreed to. :param user_id: Matrix user ID to fetch the URLs for. :return: A list of the URLs of the terms accepted by the user. """ cur = self.sydent.db.cursor() res = cur.execute( "select url from accepted_terms_urls " "where user_id = ?", (user_id,), ) urls = [] for (url,) in res: # Ensure we're dealing with unicode. if url and isinstance(url, bytes): url = url.decode("UTF-8") urls.append(url) return urls def addAgreedUrls(self, user_id: str, urls: List[str]) -> None: """ Saves that the given user has accepted the terms at the given URLs. :param user_id: The Matrix user ID that has accepted the terms. :param urls: The list of URLs. """ cur = self.sydent.db.cursor() cur.executemany( "insert or ignore into accepted_terms_urls (user_id, url) values (?, ?)", ((user_id, u) for u in urls), ) self.sydent.db.commit() sydent-2.5.1/sydent/db/threepid_associations.py000066400000000000000000000412751414516477000216770ustar00rootroot00000000000000# Copyright 2014,2017 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer from sydent.util import time_msec if TYPE_CHECKING: from sydent.sydent import Sydent # Key: id from associations db table # Value: an association dict. Roughly speaking, a signed # version of sydent.db.TheepidAssociation. SignedAssociations = Dict[int, Dict[str, Any]] logger = logging.getLogger(__name__) class LocalAssociationStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: """ Updates an association, or creates one if none exists with these parameters. Please note that email addresses in the association should be casefolded before calling this function :param assoc: The association to create or update. """ cur = self.sydent.db.cursor() # sqlite's support for upserts is atrocious cur.execute( "insert or replace into local_threepid_associations " "('medium', 'address', 'lookup_hash', 'mxid', 'ts', 'notBefore', 'notAfter')" " values (?, ?, ?, ?, ?, ?, ?)", ( assoc.medium, assoc.address, assoc.lookup_hash, assoc.mxid, assoc.ts, assoc.not_before, assoc.not_after, ), ) self.sydent.db.commit() def getAssociationsAfterId( self, afterId: Optional[int], limit: Optional[int] = None ) -> Tuple[Dict[int, ThreepidAssociation], Optional[int]]: """ Retrieves every association after the given ID. :param afterId: The ID after which to retrieve associations. :param limit: The maximum number of associations to retrieve, or None if no limit. :return: The retrieved associations (in a dict[id, assoc]), and the highest ID retrieved (or None if no ID thus no association was retrieved). """ cur = self.sydent.db.cursor() if afterId is None: afterId = -1 q = ( "select id, medium, address, lookup_hash, mxid, ts, notBefore, notAfter from " "local_threepid_associations " "where id > ? order by id asc" ) if limit is not None: q += " limit ?" res = cur.execute(q, (afterId, limit)) else: # No no, no no no no, no no no no, no no, there's no limit. res = cur.execute(q, (afterId,)) maxId = None assocs = {} row: Tuple[ int, str, str, Optional[str], Optional[str], Optional[int], Optional[int], Optional[int], ] for row in res.fetchall(): assoc = ThreepidAssociation( row[1], row[2], row[3], row[4], row[5], row[6], row[7] ) assocs[row[0]] = assoc maxId = row[0] return assocs, maxId def getSignedAssociationsAfterId( self, afterId: Optional[int], limit: Optional[int] = None ) -> Tuple[SignedAssociations, Optional[int]]: """Get associations after a given ID, and sign them before returning :param afterId: The ID to return results after (not inclusive) :param limit: The maximum amount of signed associations to return. None for no limit. :return: A tuple consisting of a dictionary containing the signed associations (id: assoc dict) and an int representing the maximum ID (which is None if there was no association to retrieve). """ assocs = {} (localAssocs, maxId) = self.getAssociationsAfterId(afterId, limit) signer = Signer(self.sydent) for localId in localAssocs: sgAssoc = signer.signedThreePidAssociation(localAssocs[localId]) assocs[localId] = sgAssoc return assocs, maxId def removeAssociation(self, threepid: Dict[str, str], mxid: str) -> None: """ Delete the association between a 3PID and a MXID, if it exists. If the association doesn't exist, log and do nothing. Please note that email addresses must be casefolded before calling this function. :param threepid: The 3PID of the binding to remove. :param mxid: The MXID of the binding to remove. """ cur = self.sydent.db.cursor() # check to see if we have any matching associations first. # We use a REPLACE INTO because we need the resulting row to have # a new ID (such that we know it's a new change that needs to be # replicated) so there's no need to insert a deletion row if there's # nothing to delete. cur.execute( "SELECT COUNT(*) FROM local_threepid_associations " "WHERE medium = ? AND address = ? AND mxid = ?", (threepid["medium"], threepid["address"], mxid), ) row: Tuple[int] = cur.fetchone() if row[0] > 0: ts = time_msec() cur.execute( "REPLACE INTO local_threepid_associations " "('medium', 'address', 'mxid', 'ts', 'notBefore', 'notAfter') " " values (?, ?, NULL, ?, null, null)", (threepid["medium"], threepid["address"], ts), ) logger.info( "Deleting local assoc for %s/%s/%s replaced %d rows", threepid["medium"], threepid["address"], mxid, cur.rowcount, ) self.sydent.db.commit() else: logger.info( "No local assoc found for %s/%s/%s", threepid["medium"], threepid["address"], mxid, ) # we still consider this successful in the name of idempotency: # the binding to be deleted is not there, so we're in the desired state. class GlobalAssociationStore: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def signedAssociationStringForThreepid( self, medium: str, address: str ) -> Optional[str]: """ Retrieve the JSON for the signed association matching the provided 3PID, if one exists. :param medium: The medium of the 3PID. :param address: The address of the 3PID. :return: The signed association, or None if no association was found for this 3PID. """ cur = self.sydent.db.cursor() # We treat address as case-insensitive because that's true for all the # threepids we have currently (we treat the local part of email addresses as # case insensitive which is technically incorrect). If we someday get a # case-sensitive threepid, this can change. res = cur.execute( "select sgAssoc from global_threepid_associations where " "medium = ? and lower(address) = lower(?) and notBefore < ? and notAfter > ? " "order by ts desc limit 1", (medium, address, time_msec(), time_msec()), ) row: Optional[Tuple[str]] = res.fetchone() if not row: return None sgAssocStr = row[0] return sgAssocStr def getMxid(self, medium: str, normalised_address: str) -> Optional[str]: """ Retrieves the MXID associated with a 3PID. Please note that emails need to be casefolded before calling this function. :param medium: The medium of the 3PID. :param normalised_address: The address of the 3PID. :return: The associated MXID, or None if no MXID is associated with this 3PID. """ cur = self.sydent.db.cursor() res = cur.execute( "select mxid from global_threepid_associations where " "medium = ? and lower(address) = lower(?) and notBefore < ? and notAfter > ? " "order by ts desc limit 1", (medium, normalised_address, time_msec(), time_msec()), ) row: Tuple[Optional[str]] = res.fetchone() if not row: return None return row[0] def getMxids( self, threepid_tuples: List[Tuple[str, str]] ) -> List[Tuple[str, str, str]]: """Given a list of threepid_tuples, return the same list but with mxids appended to each tuple for which a match was found in the database for. Output is ordered by medium, address, timestamp DESC :param threepid_tuples: List containing (medium, address) tuples :return: a list of (medium, address, mxid) tuples """ cur = self.sydent.db.cursor() cur.execute( "CREATE TEMPORARY TABLE tmp_getmxids (medium VARCHAR(16), address VARCHAR(256))" ) cur.execute( "CREATE INDEX tmp_getmxids_medium_lower_address ON tmp_getmxids (medium, lower(address))" ) try: inserted_cap = 0 while inserted_cap < len(threepid_tuples): cur.executemany( "INSERT INTO tmp_getmxids (medium, address) VALUES (?, ?)", threepid_tuples[inserted_cap : inserted_cap + 500], ) inserted_cap += 500 res = cur.execute( # 'notBefore' is the time the association starts being valid, 'notAfter' the the time at which # it ceases to be valid, so the ts must be greater than 'notBefore' and less than 'notAfter'. "SELECT gte.medium, gte.address, gte.ts, gte.mxid FROM global_threepid_associations gte " "JOIN tmp_getmxids ON gte.medium = tmp_getmxids.medium AND lower(gte.address) = lower(tmp_getmxids.address) " "WHERE gte.notBefore < ? AND gte.notAfter > ? " "ORDER BY gte.medium, gte.address, gte.ts DESC", (time_msec(), time_msec()), ) results = [] current = None row: Tuple[str, str, int, str] for row in res.fetchall(): # only use the most recent entry for each # threepid (they're sorted by ts) if (row[0], row[1]) == current: continue current = (row[0], row[1]) results.append((row[0], row[1], row[3])) finally: cur.execute("DROP TABLE tmp_getmxids") return results def addAssociation( self, assoc: ThreepidAssociation, rawSgAssoc: str, originServer: str, originId: int, commit: bool = True, ) -> None: """ Saves an association received through either a replication push or a local push. Please note that emails in the association need to be casefolded before calling this function. :param assoc: The association to add as a high level object. :param rawSgAssoc: The original raw text of the signed association. :param originServer: The name of the server the association was created on. :param originId: The ID of the association on the server the association was created on. :param commit: Whether to commit the database transaction after inserting the association. """ cur = self.sydent.db.cursor() cur.execute( "insert or ignore into global_threepid_associations " "(medium, address, lookup_hash, mxid, ts, notBefore, notAfter, originServer, originId, sgAssoc) values " "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( assoc.medium, assoc.address, assoc.lookup_hash, assoc.mxid, assoc.ts, assoc.not_before, assoc.not_after, originServer, originId, rawSgAssoc, ), ) if commit: self.sydent.db.commit() def lastIdFromServer(self, server: str) -> Optional[int]: """ Retrieves the ID of the last association received from the given peer. :param server: :return: The the ID of the last association received from the peer, or None if no association has ever been received from that peer. """ cur = self.sydent.db.cursor() res = cur.execute( "select max(originId),count(originId) from global_threepid_associations " "where originServer = ?", (server,), ) row: Tuple[int, int] = res.fetchone() if row[1] == 0: return None return row[0] def removeAssociation(self, medium: str, normalised_address: str) -> None: """ Removes any association stored for the provided 3PID. Please note that email addresses must be casefolded before calling this function. :param medium: The medium for the 3PID. :param normalised_address: The address for the 3PID. """ cur = self.sydent.db.cursor() cur.execute( "DELETE FROM global_threepid_associations WHERE " "medium = ? AND address = ?", (medium, normalised_address), ) logger.info( "Deleted %d rows from global associations for %s/%s", cur.rowcount, medium, normalised_address, ) self.sydent.db.commit() def retrieveMxidsForHashes(self, addresses: List[str]) -> Dict[str, str]: """Returns a mapping from hash: mxid from a list of given lookup_hash values :param addresses: An array of lookup_hash values to check against the db :returns a dictionary of lookup_hash values to mxids of all discovered matches """ cur = self.sydent.db.cursor() cur.execute( "CREATE TEMPORARY TABLE tmp_retrieve_mxids_for_hashes " "(lookup_hash VARCHAR)" ) cur.execute( "CREATE INDEX tmp_retrieve_mxids_for_hashes_lookup_hash ON " "tmp_retrieve_mxids_for_hashes(lookup_hash)" ) results = {} try: # Convert list of addresses to list of tuples of addresses tuplized_addresses = [(x,) for x in addresses] inserted_cap = 0 while inserted_cap < len(tuplized_addresses): cur.executemany( "INSERT INTO tmp_retrieve_mxids_for_hashes(lookup_hash) " "VALUES (?)", tuplized_addresses[inserted_cap : inserted_cap + 500], ) inserted_cap += 500 res = cur.execute( # 'notBefore' is the time the association starts being valid, 'notAfter' the the time at which # it ceases to be valid, so the ts must be greater than 'notBefore' and less than 'notAfter'. "SELECT gta.lookup_hash, gta.mxid FROM global_threepid_associations gta " "JOIN tmp_retrieve_mxids_for_hashes " "ON gta.lookup_hash = tmp_retrieve_mxids_for_hashes.lookup_hash " "WHERE gta.notBefore < ? AND gta.notAfter > ? " "ORDER BY gta.lookup_hash, gta.mxid, gta.ts", (time_msec(), time_msec()), ) # Place the results from the query into a dictionary # Results are sorted from oldest to newest, so if there are multiple mxid's for # the same lookup hash, only the newest mapping will be returned # Type safety: lookup_hash is a nullable string in # global_threepid_associations. But it must be equal to a lookup_hash # in the temporary table thanks to the join condition. # The temporary table gets hashes from the `addresses` argument, # which is a list of (non-None) strings. # So lookup_hash really is a str. lookup_hash: str mxid: str for lookup_hash, mxid in res.fetchall(): results[lookup_hash] = mxid finally: cur.execute("DROP TABLE tmp_retrieve_mxids_for_hashes") return results sydent-2.5.1/sydent/db/threepid_associations.sql000066400000000000000000000026241414516477000220410ustar00rootroot00000000000000/* Copyright 2014,2017 OpenMarket Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -- Note that this SQL file is not up to date, and migrations can be found in sydent/db/sqlitedb.py CREATE TABLE IF NOT EXISTS local_threepid_associations ( id integer primary key, medium varchar(16) not null, address varchar(256) not null, mxid varchar(256) not null, ts integer not null, notBefore bigint not null, notAfter bigint not null ); CREATE TABLE IF NOT EXISTS global_threepid_associations ( id integer primary key, medium varchar(16) not null, address varchar(256) not null, mxid varchar(256) not null, ts integer not null, notBefore bigint not null, notAfter integer not null, originServer varchar(255) not null, originId integer not null, sgAssoc text not null ); CREATE UNIQUE INDEX IF NOT EXISTS originServer_originId on global_threepid_associations (originServer, originId); sydent-2.5.1/sydent/db/threepid_validation.sql000066400000000000000000000021521414516477000214700ustar00rootroot00000000000000/* Copyright 2014 OpenMarket Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -- Note that this SQL file is not up to date, and migrations can be found in sydent/db/sqlitedb.py CREATE TABLE IF NOT EXISTS threepid_validation_sessions (id integer primary key, medium varchar(16) not null, address varchar(256) not null, clientSecret varchar(32) not null, validated int default 0, mtime bigint not null); CREATE TABLE IF NOT EXISTS threepid_token_auths (id integer primary key, validationSession integer not null, token varchar(32) not null, sendAttemptNumber integer not null, foreign key (validationSession) references threepid_validations(id)); sydent-2.5.1/sydent/db/valsession.py000066400000000000000000000225421414516477000174760ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from random import SystemRandom from typing import TYPE_CHECKING, Optional, Tuple import sydent.util.tokenutils from sydent.util import time_msec from sydent.validators import ( THREEPID_SESSION_VALID_LIFETIME_MS, IncorrectClientSecretException, InvalidSessionIdException, SessionExpiredException, SessionNotValidatedException, TokenInfo, ValidationSession, ) if TYPE_CHECKING: from sydent.sydent import Sydent class ThreePidValSessionStore: def __init__(self, syd: "Sydent") -> None: self.sydent = syd self.random = SystemRandom() def getOrCreateTokenSession( self, medium: str, address: str, clientSecret: str ) -> Tuple[ValidationSession, TokenInfo]: """ Retrieves the validation session for a given medium, address and client secret, or creates one if none was found. :param medium: The medium to use when looking up or creating the session. :param address: The address to use when looking up or creating the session. :param clientSecret: The client secret to use when looking up or creating the session. :return: The session that was retrieved or created. """ cur = self.sydent.db.cursor() cur.execute( "select s.id, s.medium, s.address, s.clientSecret, s.validated, s.mtime, " "t.token, t.sendAttemptNumber from threepid_validation_sessions s,threepid_token_auths t " "where s.medium = ? and s.address = ? and s.clientSecret = ? and t.validationSession = s.id", (medium, address, clientSecret), ) row: Optional[ Tuple[int, str, str, str, Optional[int], int, str, int] ] = cur.fetchone() if row: session = ValidationSession( row[0], row[1], row[2], row[3], bool(row[4]), row[5] ) token_info = TokenInfo(row[6], row[7]) return session, token_info sid = self.addValSession( medium, address, clientSecret, time_msec(), commit=False ) tokenString = sydent.util.tokenutils.generateTokenForMedium(medium) cur.execute( "insert into threepid_token_auths (validationSession, token, sendAttemptNumber) values (?, ?, ?)", (sid, tokenString, -1), ) self.sydent.db.commit() session = ValidationSession( sid, medium, address, clientSecret, False, time_msec(), ) token_info = TokenInfo(tokenString, -1) return session, token_info def addValSession( self, medium: str, address: str, clientSecret: str, mtime: int, commit: bool = True, ) -> int: """ Creates a validation session with the given parameters. :param medium: The medium to create the session for. :param address: The address to create the session for. :param clientSecret: The client secret to use when looking up or creating the session. :param mtime: The current time in milliseconds. :param commit: Whether to commit the transaction after executing the insert statement. :return: The ID of the created session. """ cur = self.sydent.db.cursor() # Let's make up a random sid rather than using sequential ones. This # should be safe enough given we reap old sessions. sid = self.random.randint(0, 2 ** 31) cur.execute( "insert into threepid_validation_sessions ('id', 'medium', 'address', 'clientSecret', 'mtime')" + " values (?, ?, ?, ?, ?)", (sid, medium, address, clientSecret, mtime), ) if commit: self.sydent.db.commit() return sid def setSendAttemptNumber(self, sid: int, attemptNo: int) -> None: """ Updates the send attempt number for the session with the given ID. :param sid: The ID of the session to update :param attemptNo: The send attempt number to update the session with. """ cur = self.sydent.db.cursor() cur.execute( "update threepid_token_auths set sendAttemptNumber = ? where id = ?", (attemptNo, sid), ) self.sydent.db.commit() def setValidated(self, sid: int, validated: bool) -> None: """ Updates a session to set the validated flag to the given value. :param sid: The ID of the session to update. :param validated: The value to set the validated flag. """ cur = self.sydent.db.cursor() cur.execute( "update threepid_validation_sessions set validated = ? where id = ?", (validated, sid), ) self.sydent.db.commit() def setMtime(self, sid: int, mtime: int) -> None: """ Set the time of the last send attempt for the session with the given ID :param sid: The ID of the session to update. :param mtime: The time of the last send attempt for that session. """ cur = self.sydent.db.cursor() cur.execute( "update threepid_validation_sessions set mtime = ? where id = ?", (mtime, sid), ) self.sydent.db.commit() def getSessionById(self, sid: int) -> Optional[ValidationSession]: """ Retrieves the session matching the given sid. :param sid: The ID of the session to retrieve. :return: The retrieved session, or None if no session could be found with that sid. """ cur = self.sydent.db.cursor() cur.execute( "select id, medium, address, clientSecret, validated, mtime from " + "threepid_validation_sessions where id = ?", (sid,), ) row: Optional[Tuple[int, str, str, str, Optional[int], int]] = cur.fetchone() if not row: return None return ValidationSession(row[0], row[1], row[2], row[3], bool(row[4]), row[5]) def getTokenSessionById( self, sid: int ) -> Optional[Tuple[ValidationSession, TokenInfo]]: """ Retrieves a validation session using the session's ID. :param sid: The ID of the session to retrieve. :return: The validation session, or None if no session was found with that ID. """ cur = self.sydent.db.cursor() cur.execute( "select s.id, s.medium, s.address, s.clientSecret, s.validated, s.mtime, " "t.token, t.sendAttemptNumber from threepid_validation_sessions s,threepid_token_auths t " "where s.id = ? and t.validationSession = s.id", (sid,), ) row: Optional[Tuple[int, str, str, str, Optional[int], int, str, int]] row = cur.fetchone() if row: s = ValidationSession(row[0], row[1], row[2], row[3], bool(row[4]), row[5]) t = TokenInfo(row[6], row[7]) return s, t return None def getValidatedSession(self, sid: int, client_secret: str) -> ValidationSession: """ Retrieve a validated and still-valid session whose client secret matches the one passed in. :param sid: The ID of the session to retrieve. :param client_secret: A client secret to check against the one retrieved from the database. :return: The retrieved session. :raise InvalidSessionIdException: No session could be found with this ID. :raise IncorrectClientSecretException: The session's client secret doesn't match the one passed in. :raise SessionExpiredException: The session exists but has expired. :raise SessionNotValidatedException: The session exists but hasn't been validated yet. """ s = self.getSessionById(sid) if not s: raise InvalidSessionIdException() if not s.client_secret == client_secret: raise IncorrectClientSecretException() if s.mtime + THREEPID_SESSION_VALID_LIFETIME_MS < time_msec(): raise SessionExpiredException() if not s.validated: raise SessionNotValidatedException() return s def deleteOldSessions(self) -> None: """Delete old threepid validation sessions that are long expired.""" cur = self.sydent.db.cursor() delete_before_ts = time_msec() - 5 * THREEPID_SESSION_VALID_LIFETIME_MS sql = """ DELETE FROM threepid_validation_sessions WHERE mtime < ? """ cur.execute(sql, (delete_before_ts,)) sql = """ DELETE FROM threepid_token_auths WHERE validationSession NOT IN ( SELECT id FROM threepid_validation_sessions ) """ cur.execute(sql) self.sydent.db.commit() sydent-2.5.1/sydent/hs_federation/000077500000000000000000000000001414516477000171565ustar00rootroot00000000000000sydent-2.5.1/sydent/hs_federation/__init__.py000066400000000000000000000000001414516477000212550ustar00rootroot00000000000000sydent-2.5.1/sydent/hs_federation/types.py000066400000000000000000000013121414516477000206710ustar00rootroot00000000000000from typing import Dict import attr from typing_extensions import TypedDict from sydent.types import JsonDict class VerifyKey(TypedDict): key: str VerifyKeys = Dict[str, VerifyKey] @attr.s(frozen=True, slots=True, auto_attribs=True) class CachedVerificationKeys: verify_keys: VerifyKeys valid_until_ts: int # key: "signing key identifier"; value: signature encoded as unpadded base 64 # See https://spec.matrix.org/unstable/appendices/#signing-details Signature = Dict[str, str] @attr.s(frozen=True, slots=True, auto_attribs=True) class SignedMatrixRequest: method: bytes uri: bytes destination_is: str signatures: Dict[str, Signature] origin: str content: JsonDict sydent-2.5.1/sydent/hs_federation/verifier.py000066400000000000000000000241421414516477000213460ustar00rootroot00000000000000# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import time from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast import attr import signedjson.key import signedjson.sign from signedjson.sign import SignatureVerifyException from twisted.web.server import Request from unpaddedbase64 import decode_base64 from sydent.hs_federation.types import ( CachedVerificationKeys, SignedMatrixRequest, VerifyKeys, ) from sydent.http.httpclient import FederationHttpClient from sydent.types import JsonDict from sydent.util.stringutils import is_valid_matrix_server_name if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class NoAuthenticationError(Exception): """ Raised when no signature is provided that could be authenticated """ pass class InvalidServerName(Exception): """ Raised when the provided origin parameter is not a valid hostname (plus optional port). """ pass class Verifier: """ Verifies signed json blobs from Matrix Homeservers by finding the homeserver's address, contacting it, requesting its keys and verifying that the signature on the json blob matches. """ def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent # Cache of server keys. These are cached until the 'valid_until_ts' time # in the result. self.cache: Dict[str, CachedVerificationKeys] = { # server_name: , } async def _getKeysForServer(self, server_name: str) -> VerifyKeys: """Get the signing key data from a homeserver. :param server_name: The name of the server to request the keys from. :return: The verification keys returned by the server. """ if server_name in self.cache: cached = self.cache[server_name] now = int(time.time() * 1000) if cached.valid_until_ts > now: return cached.verify_keys client = FederationHttpClient(self.sydent) # Cast safety: we have validation logic below which checks that # - `verify_keys` is present # - `valid_until_ts` is an integer if present # - cached entries always have a `valid_until_ts` key # and we don't use any of the other fields on GetKeyResponse. # The only use of the cache is in this function, and only to read # the two fields mentioned above. result = await client.get_json( "matrix://%s/_matrix/key/v2/server/" % server_name, 1024 * 50 ) if "verify_keys" not in result: raise SignatureVerifyException("No key found in response") if not isinstance(result["verify_keys"], dict): raise SignatureVerifyException( f"Invalid type for verify_keys: expected dict, got {result['verify_keys']}" ) keys_to_remove = [] for key_name, key_dict in result["verify_keys"].items(): if "key" not in key_dict: logger.warning("Ignoring key %s with no 'key'", key_name) keys_to_remove.append(key_name) elif not isinstance(key_dict["key"], str): raise SignatureVerifyException( f"Invalid type for verify_keys/{key_name}/key: " f"expected str, got {key_dict['key']}" ) for key_name in keys_to_remove: del result["verify_keys"][key_name] # We've now verified that verify_keys has the correct type. verify_keys: VerifyKeys = result["verify_keys"] if "valid_until_ts" in result: if not isinstance(result["valid_until_ts"], int): raise SignatureVerifyException( "Invalid valid_until_ts received, must be an integer" ) # Don't cache anything without a valid_until_ts or we wouldn't # know when to expire it. logger.info( "Got keys for %s: caching until %d", server_name, result["valid_until_ts"], ) self.cache[server_name] = CachedVerificationKeys( verify_keys, result["valid_until_ts"] ) return verify_keys async def verifyServerSignedJson( self, signed_json: SignedMatrixRequest, acceptable_server_names: Optional[List[str]] = None, ) -> Tuple[str, str]: """Given a signed json object, try to verify any one of the signatures on it XXX: This contains a fairly noddy version of the home server SRV lookup and signature verification. It does no caching (just fetches the signature each time and does not contact any other servers to do perspective checks). :param acceptable_server_names: If provided and not None, only signatures from servers in this list will be accepted. :return a tuple of the server name and key name that was successfully verified. :raise SignatureVerifyException: The json cannot be verified. """ for server_name, sigs in signed_json.signatures.items(): if acceptable_server_names is not None: if server_name not in acceptable_server_names: continue server_keys = await self._getKeysForServer(server_name) for key_name, sig in sigs.items(): if key_name in server_keys: key_bytes = decode_base64(server_keys[key_name]["key"]) verify_key = signedjson.key.decode_verify_key_bytes( key_name, key_bytes ) logger.info("verifying sig from key %r", key_name) payload = attr.asdict(signed_json) signedjson.sign.verify_signed_json(payload, server_name, verify_key) logger.info( "Verified signature with key %s from %s", key_name, server_name ) return (server_name, key_name) logger.warning( "No matching key found for signature block %r in server keys %r", signed_json.signatures, server_keys, ) logger.warning( "Unable to verify any signatures from block %r. Acceptable server names: %r", signed_json.signatures, acceptable_server_names, ) raise SignatureVerifyException("No matching signature found") async def authenticate_request(self, request: "Request", content: JsonDict) -> str: """Authenticates a Matrix federation request based on the X-Matrix header XXX: Copied largely from synapse :param request: The request object to authenticate :param content: The content of the request, if any :return: The origin of the server whose signature was validated """ auth_headers = request.requestHeaders.getRawHeaders("Authorization") if not auth_headers: raise NoAuthenticationError("Missing Authorization headers") # Retrieve an origin and signatures from the authorization header. origin = None signatures: Dict[str, Dict[str, str]] = {} for auth in auth_headers: if auth.startswith("X-Matrix"): (origin, key, sig) = parse_auth_header(auth) signatures.setdefault(origin, {})[key] = sig if origin is None: raise NoAuthenticationError("Missing X-Matrix Authorization header") if not is_valid_matrix_server_name(origin): raise InvalidServerName( "X-Matrix header's origin parameter must be a valid Matrix server name" ) json_request = SignedMatrixRequest( method=request.method, uri=request.uri, destination_is=self.sydent.config.general.server_name, signatures=signatures, origin=origin, content=content, ) await self.verifyServerSignedJson(json_request, [origin]) logger.info("Verified request from HS %s", origin) return origin def parse_auth_header(header_str: str) -> Tuple[str, str, str]: """ Extracts a server name, signing key and payload signature from an "Authorization: X-Matrix ..." header. :param header_str: The content of the header, Starting at "X-Matrix". For example, `X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..."` See https://matrix.org/docs/spec/server_server/r0.1.4#request-authentication :return: The server name, the signing key, and the payload signature. :raises SignatureVerifyException: if the header did not meet the expected format. """ try: # Strip off "X-Matrix " and break up into key-value pairs. params = header_str.split(" ")[1].split(",") param_dict: Dict[str, str] = dict( # Cast safety: the split() call will either return a 1- or 2- tuple. # If it returns a 1-tuple, dict() will complain with a ValueError # so we'll spot the bad header. cast(Tuple[str, str], kv.split("=", maxsplit=1)) for kv in params ) def strip_quotes(value: str) -> str: if value.startswith('"'): return value[1:-1] else: return value origin = strip_quotes(param_dict["origin"]) key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) return origin, key, sig except Exception: raise SignatureVerifyException("Malformed Authorization header") sydent-2.5.1/sydent/http/000077500000000000000000000000001414516477000153235ustar00rootroot00000000000000sydent-2.5.1/sydent/http/__init__.py000066400000000000000000000000001414516477000174220ustar00rootroot00000000000000sydent-2.5.1/sydent/http/auth.py000066400000000000000000000054621414516477000166450ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, Optional from twisted.web.server import Request from sydent.db.accounts import AccountStore from sydent.http.servlets import MatrixRestError, get_args from sydent.terms.terms import get_terms if TYPE_CHECKING: from sydent.sydent import Sydent from sydent.users.accounts import Account logger = logging.getLogger(__name__) def tokenFromRequest(request: Request) -> Optional[str]: """Extract token from header of query parameter. :param request: The request to look for an access token in. :return: The token or None if not found """ token = None # check for Authorization header first authHeader = request.getHeader("Authorization") if authHeader is not None and authHeader.startswith("Bearer "): token = authHeader[len("Bearer ") :] # no? try access_token query param if token is None: args = get_args(request, ("access_token",), required=False) token = args.get("access_token") return token def authV2( sydent: "Sydent", request: Request, requireTermsAgreed: bool = True, ) -> "Account": """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. :param request: The request to look for an access token in. :param requireTermsAgreed: Whether to deny authentication if the user hasn't accepted the terms of service. :returns Account: The account object if there is correct auth :raises MatrixRestError: If the request is v2 but could not be authed or the user has not accepted terms. """ token = tokenFromRequest(request) if token is None: raise MatrixRestError(401, "M_UNAUTHORIZED", "Unauthorized") accountStore = AccountStore(sydent) account = accountStore.getAccountByToken(token) if account is None: raise MatrixRestError(401, "M_UNAUTHORIZED", "Unauthorized") if requireTermsAgreed: terms = get_terms(sydent) if ( terms.getMasterVersion() is not None and account.consentVersion != terms.getMasterVersion() ): raise MatrixRestError(403, "M_TERMS_NOT_SIGNED", "Terms not signed") return account sydent-2.5.1/sydent/http/blacklisting_reactor.py000066400000000000000000000114351414516477000220660ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import Any, List, Optional from netaddr import IPAddress, IPSet from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.interfaces import ( IAddress, IHostResolution, IReactorPluggableNameResolver, IResolutionReceiver, ) from zope.interface import implementer, provider logger = logging.getLogger(__name__) def check_against_blacklist( ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet ) -> bool: """ Compares an IP address to allowed and disallowed IP sets. Args: ip_address: The IP address to check ip_whitelist: Allowed IP addresses. ip_blacklist: Disallowed IP addresses. Returns: True if the IP address is in the blacklist and not in the whitelist. """ if ip_address in ip_blacklist: if ip_whitelist is None or ip_address not in ip_whitelist: return True return False class _IPBlacklistingResolver: """ A proxy for reactor.nameResolver which only produces non-blacklisted IP addresses, preventing DNS rebinding attacks on URL preview. """ def __init__( self, reactor: IReactorPluggableNameResolver, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet, ): """ Args: reactor: The twisted reactor. ip_whitelist: IP addresses to allow. ip_blacklist: IP addresses to disallow. """ self._reactor = reactor self._ip_whitelist = ip_whitelist self._ip_blacklist = ip_blacklist def resolveHostName( self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0 ) -> IResolutionReceiver: addresses = [] # type: List[IAddress] def _callback() -> None: has_bad_ip = False for address in addresses: # We only expect IPv4 and IPv6 addresses since only A/AAAA lookups # should go through this path. if not isinstance(address, (IPv4Address, IPv6Address)): continue ip_address = IPAddress(address.host) if check_against_blacklist( ip_address, self._ip_whitelist, self._ip_blacklist ): logger.info( "Dropped %s from DNS resolution to %s due to blacklist" % (ip_address, hostname) ) has_bad_ip = True # if we have a blacklisted IP, we'd like to raise an error to block the # request, but all we can really do from here is claim that there were no # valid results. if not has_bad_ip: for address in addresses: recv.addressResolved(address) recv.resolutionComplete() @provider(IResolutionReceiver) class EndpointReceiver: @staticmethod def resolutionBegan(resolutionInProgress: IHostResolution) -> None: recv.resolutionBegan(resolutionInProgress) @staticmethod def addressResolved(address: IAddress) -> None: addresses.append(address) @staticmethod def resolutionComplete() -> None: _callback() self._reactor.nameResolver.resolveHostName( EndpointReceiver, hostname, portNumber=portNumber ) return recv @implementer(IReactorPluggableNameResolver) class BlacklistingReactorWrapper: """ A Reactor wrapper which will prevent DNS resolution to blacklisted IP addresses, to prevent DNS rebinding. """ def __init__( self, reactor: IReactorPluggableNameResolver, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet, ): self._reactor = reactor # We need to use a DNS resolver which filters out blacklisted IP # addresses, to prevent DNS rebinding. self.nameResolver = _IPBlacklistingResolver( self._reactor, ip_whitelist, ip_blacklist ) def __getattr__(self, attr: str) -> Any: # Passthrough to the real reactor except for the DNS resolver. return getattr(self._reactor, attr) sydent-2.5.1/sydent/http/federation_tls_options.py000066400000000000000000000076431414516477000224640ustar00rootroot00000000000000# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import Callable from OpenSSL import SSL from twisted.internet import ssl from twisted.internet.abstract import isIPAddress, isIPv6Address from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.protocols.tls import TLSMemoryBIOProtocol from twisted.python.failure import Failure from zope.interface import implementer logger = logging.getLogger(__name__) F = Callable[[SSL.Connection, int, int], None] def _tolerateErrors(wrapped: F) -> F: """ Wrap up an info_callback for pyOpenSSL so that if something goes wrong the error is immediately logged and the connection is dropped if possible. This is a copy of twisted.internet._sslverify._tolerateErrors. For documentation, see the twisted documentation. """ def infoCallback(connection: SSL.Connection, where: int, ret: int) -> None: try: return wrapped(connection, where, ret) except BaseException: f = Failure() logger.exception("Error during info_callback") connection.get_app_data().failVerification(f) return infoCallback def _idnaBytes(text: str) -> bytes: """ Convert some text typed by a human into some ASCII bytes. This is a copy of twisted.internet._idna._idnaBytes. For documentation, see the twisted documentation. """ try: import idna except ImportError: return text.encode("idna") else: return idna.encode(text) @implementer(IOpenSSLClientConnectionCreator) class ClientTLSOptions: """ Client creator for TLS without certificate identity verification. This is a copy of twisted.internet._sslverify.ClientTLSOptions with the identity verification left out. For documentation, see the twisted documentation. """ def __init__(self, hostname: str, ctx: SSL.Context): self._ctx = ctx if isIPAddress(hostname) or isIPv6Address(hostname): self._hostnameBytes = hostname.encode("ascii") self._sendSNI = False else: self._hostnameBytes = _idnaBytes(hostname) self._sendSNI = True ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback)) def clientConnectionForTLS( self, tlsProtocol: TLSMemoryBIOProtocol ) -> SSL.Connection: context = self._ctx connection = SSL.Connection(context, None) connection.set_app_data(tlsProtocol) return connection def _identityVerifyingInfoCallback( self, connection: SSL.Connection, where: int, ret: int ) -> None: # Literal IPv4 and IPv6 addresses are not permitted # as host names according to the RFCs if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI: connection.set_tlsext_host_name(self._hostnameBytes) class ClientTLSOptionsFactory: """Factory for Twisted ClientTLSOptions that are used to make connections to remote servers for federation.""" def __init__(self, verify_requests: bool): if verify_requests: self._options = ssl.CertificateOptions(trustRoot=ssl.platformTrust()) else: self._options = ssl.CertificateOptions() def get_options(self, host: str) -> ClientTLSOptions: # Use _makeContext so that we get a fresh OpenSSL CTX each time. return ClientTLSOptions(host, self._options._makeContext()) sydent-2.5.1/sydent/http/httpclient.py000066400000000000000000000155171414516477000200640ustar00rootroot00000000000000# Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import logging from io import BytesIO from typing import TYPE_CHECKING, Any, Dict, Generic, Optional, Tuple, TypeVar, cast from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent, IResponse from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper from sydent.http.federation_tls_options import ClientTLSOptionsFactory from sydent.http.httpcommon import read_body_with_max_size from sydent.http.matrixfederationagent import MatrixFederationAgent from sydent.types import JsonDict from sydent.util import json_decoder if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) AgentType = TypeVar("AgentType", bound=IAgent) class HTTPClient(Generic[AgentType]): """A base HTTP class that contains methods for making GET and POST HTTP requests. """ agent: AgentType async def get_json(self, uri: str, max_size: Optional[int] = None) -> JsonDict: """Make a GET request to an endpoint returning JSON and parse result :param uri: The URI to make a GET request to. :param max_size: The maximum size (in bytes) to allow as a response. :return: A deferred containing JSON parsed into a Python object. """ logger.debug("HTTP GET %s", uri) response = await self.agent.request( b"GET", uri.encode("utf8"), ) body = await read_body_with_max_size(response, max_size) try: # json.loads doesn't allow bytes in Python 3.5 json_body = json_decoder.decode(body.decode("UTF-8")) except Exception: logger.warning("Error parsing JSON from %s", uri) raise if not isinstance(json_body, dict): raise TypeError # Cast safety: json only permits strings as object keys, so `json_body` # must be Dict[str, Any] rather than Dict[Any, Any]. return cast(JsonDict, json_body) async def post_json_get_nothing( self, uri: str, post_json: JsonDict, opts: Dict[str, Any] ) -> IResponse: """Make a POST request to an endpoint returning nothing :param uri: The URI to make a POST request to. :param post_json: A Python object that will be converted to a JSON string and POSTed to the given URI. :param opts: A dictionary of request options. Currently only opts.headers is supported. :return: a response from the remote server. """ resp, _ = await self.post_json_maybe_get_json(uri, post_json, opts) return resp async def post_json_maybe_get_json( self, uri: str, post_json: Dict[str, Any], opts: Dict[str, Any], max_size: Optional[int] = None, ) -> Tuple[IResponse, Optional[JsonDict]]: """Make a POST request to an endpoint that might be returning JSON and parse result :param uri: The URI to make a POST request to. :param post_json: A Python object that will be converted to a JSON string and POSTed to the given URI. :param opts: A dictionary of request options. Currently only opts.headers is supported. :param max_size: The maximum size (in bytes) to allow as a response. :return: a response from the remote server, and its decoded JSON body if any (None otherwise). """ json_bytes = json.dumps(post_json).encode("utf8") headers = opts.get( "headers", Headers( { b"Content-Type": [b"application/json"], } ), ) logger.debug("HTTP POST %s -> %s", json_bytes, uri) response = await self.agent.request( b"POST", uri.encode("utf8"), headers, bodyProducer=FileBodyProducer(BytesIO(json_bytes)), ) # Ensure the body object is read otherwise we'll leak HTTP connections # as per # https://twistedmatrix.com/documents/current/web/howto/client.html json_body = None try: # TODO Will this cause the server to think the request was a failure? body = await read_body_with_max_size(response, max_size) json_body = json_decoder.decode(body.decode("UTF-8")) except Exception: # We might get an exception here because the body exceeds the max_size, or it # isn't valid JSON. In both cases, we don't care about it. pass return response, json_body class SimpleHttpClient(HTTPClient[Agent]): """A simple, no-frills HTTP client based on the class of the same name from Synapse. """ def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent # The default endpoint factory in Twisted 14.0.0 (which we require) uses the # BrowserLikePolicyForHTTPS context factory which will do regular cert validation # 'like a browser' self.agent = Agent( BlacklistingReactorWrapper( reactor=self.sydent.reactor, ip_whitelist=sydent.config.general.ip_whitelist, ip_blacklist=sydent.config.general.ip_blacklist, ), connectTimeout=15, ) class FederationHttpClient(HTTPClient[MatrixFederationAgent]): """HTTP client for federation requests to homeservers. Uses a MatrixFederationAgent. """ def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.agent = MatrixFederationAgent( # Type-safety: I don't have a good way of expressing that # the reactor is IReactorTCP, IReactorTime and # IReactorPluggableNameResolver all at once. But it is, because # it wraps the sydent reactor. # TODO: can we introduce a SydentReactor type like SynapseReactor? BlacklistingReactorWrapper( # type: ignore[arg-type] reactor=self.sydent.reactor, ip_whitelist=sydent.config.general.ip_whitelist, ip_blacklist=sydent.config.general.ip_blacklist, ), ClientTLSOptionsFactory(sydent.config.http.verify_federation_certs) if sydent.use_tls_for_federation else None, ) sydent-2.5.1/sydent/http/httpcommon.py000066400000000000000000000173671414516477000201030ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from io import BytesIO from typing import TYPE_CHECKING, Optional, cast import twisted.internet.ssl from twisted.internet import defer, protocol from twisted.internet._sslverify import IOpenSSLTrustRoot from twisted.internet.interfaces import ITCPTransport from twisted.internet.protocol import connectionDone from twisted.python.failure import Failure from twisted.web import server from twisted.web.client import ResponseDone from twisted.web.http import PotentialDataLoss from twisted.web.iweb import UNKNOWN_LENGTH, IResponse if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) # Arbitrarily limited to 512 KiB. MAX_REQUEST_SIZE = 512 * 1024 class SslComponents: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.myPrivateCertificate = self.makeMyCertificate() self.trustRoot = self.makeTrustRoot() def makeMyCertificate(self) -> Optional[twisted.internet.ssl.PrivateCertificate]: # TODO Move some of this loading into parse_config privKeyAndCertFilename = self.sydent.config.http.cert_file if privKeyAndCertFilename == "": logger.warning( "No HTTPS private key / cert found: not starting replication server " "or doing replication pushes" ) return None try: fp = open(privKeyAndCertFilename) except OSError: logger.warning( "Unable to read private key / cert file from %s: not starting the replication HTTPS server " "or doing replication pushes.", privKeyAndCertFilename, ) return None authData = fp.read() fp.close() return twisted.internet.ssl.PrivateCertificate.loadPEM(authData) def makeTrustRoot(self) -> IOpenSSLTrustRoot: # If this option is specified, use a specific root CA cert. This is useful for testing when it's not # practical to get the client cert signed by a real root CA but should never be used on a production server. caCertFilename = self.sydent.config.http.ca_cert_file if len(caCertFilename) > 0: try: fp = open(caCertFilename) caCert = twisted.internet.ssl.Certificate.loadPEM(fp.read()) fp.close() except Exception: logger.warning("Failed to open CA cert file %s", caCertFilename) raise logger.warning("Using custom CA cert file: %s", caCertFilename) # Type ignore: I'm not going to add a stub for the semiprivate # _sslverify module. I've already taken on too much stubbing as it is! return twisted.internet._sslverify.OpenSSLCertificateAuthorities( # type: ignore [caCert.original] ) else: return twisted.internet.ssl.OpenSSLDefaultPaths() class BodyExceededMaxSize(Exception): """The maximum allowed size of the HTTP body was exceeded.""" class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol): """A protocol which immediately errors upon receiving data.""" transport: ITCPTransport def __init__(self, deferred: "defer.Deferred[bytes]") -> None: self.deferred = deferred def _maybe_fail(self) -> None: """ Report a max size exceed error and disconnect the first time this is called. """ if not self.deferred.called: self.deferred.errback(BodyExceededMaxSize()) # Close the connection (forcefully) since all the data will get # discarded anyway. self.transport.abortConnection() def dataReceived(self, data: bytes) -> None: self._maybe_fail() def connectionLost(self, reason: Failure = connectionDone) -> None: self._maybe_fail() class _ReadBodyWithMaxSizeProtocol(protocol.Protocol): """A protocol which reads body to a stream, erroring if the body exceeds a maximum size.""" transport: ITCPTransport def __init__( self, deferred: "defer.Deferred[bytes]", max_size: Optional[int] ) -> None: self.stream = BytesIO() self.deferred = deferred self.length = 0 self.max_size = max_size def dataReceived(self, data: bytes) -> None: # If the deferred was called, bail early. if self.deferred.called: return self.stream.write(data) self.length += len(data) # The first time the maximum size is exceeded, error and cancel the # connection. dataReceived might be called again if data was received # in the meantime. if self.max_size is not None and self.length >= self.max_size: self.deferred.errback(BodyExceededMaxSize()) # Close the connection (forcefully) since all the data will get # discarded anyway. if self.transport is not None: self.transport.abortConnection() def connectionLost(self, reason: Failure = connectionDone) -> None: # If the maximum size was already exceeded, there's nothing to do. if self.deferred.called: return if reason.check(ResponseDone): self.deferred.callback(self.stream.getvalue()) elif reason.check(PotentialDataLoss): # stolen from https://github.com/twisted/treq/pull/49/files # http://twistedmatrix.com/trac/ticket/4840 self.deferred.callback(self.stream.getvalue()) else: self.deferred.errback(reason) def read_body_with_max_size( response: IResponse, max_size: Optional[int] ) -> "defer.Deferred[bytes]": """ Read a HTTP response body to a file-object. Optionally enforcing a maximum file size. If the maximum file size is reached, the returned Deferred will resolve to a Failure with a BodyExceededMaxSize exception. Args: response: The HTTP response to read from. max_size: The maximum file size to allow. Returns: A Deferred which resolves to the read body. """ d: "defer.Deferred[bytes]" = defer.Deferred() # If the Content-Length header gives a size larger than the maximum allowed # size, do not bother downloading the body. # Type safety: twisted guarantees that response.length is either the # "opaque" object UNKNOWN_LENGTH, or else an int. if max_size is not None and response.length != UNKNOWN_LENGTH: response.length = cast(int, response.length) if response.length > max_size: response.deliverBody(_DiscardBodyWithMaxSizeProtocol(d)) return d response.deliverBody(_ReadBodyWithMaxSizeProtocol(d, max_size)) return d class SizeLimitingRequest(server.Request): def handleContentChunk(self, data: bytes) -> None: if self.content.tell() + len(data) > MAX_REQUEST_SIZE: logger.info( "Aborting connection from %s because the request exceeds maximum size", # Formerly `self.client.host`, but `host` isn't provided by `IAddress` self.client, ) assert self.transport is not None self.transport.abortConnection() return return super().handleContentChunk(data) sydent-2.5.1/sydent/http/httpsclient.py000066400000000000000000000066401414516477000202440ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import logging from io import BytesIO from typing import TYPE_CHECKING, Optional from twisted.internet.defer import Deferred from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import optionsForClientTLS from twisted.web.client import Agent, FileBodyProducer, Response from twisted.web.http_headers import Headers from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class ReplicationHttpsClient: """ An HTTPS client specifically for talking replication to other Matrix Identity Servers (ie. presents our replication SSL certificate and validates peer SSL certificates as we would in the replication HTTPS server) """ def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.agent: Optional[Agent] = None if self.sydent.sslComponents.myPrivateCertificate: # We will already have logged a warn if this is absent, so don't do it again # cert = self.sydent.sslComponents.myPrivateCertificate # self.certOptions = twisted.internet.ssl.CertificateOptions(privateKey=cert.privateKey.original, # certificate=cert.original, # trustRoot=self.sydent.sslComponents.trustRoot) self.agent = Agent(self.sydent.reactor, SydentPolicyForHTTPS(self.sydent)) def postJson( self, uri: str, jsonObject: JsonDict ) -> Optional["Deferred[Response]"]: """ Sends an POST request over HTTPS. :param uri: The URI to send the request to. :param jsonObject: The request's body. :return: The request's response. """ logger.debug("POSTing request to %s", uri) if not self.agent: logger.error("HTTPS post attempted but HTTPS is not configured") return None headers = Headers( {"Content-Type": ["application/json"], "User-Agent": ["Sydent"]} ) json_bytes = json.dumps(jsonObject).encode("utf8") reqDeferred = self.agent.request( b"POST", uri.encode("utf8"), headers, FileBodyProducer(BytesIO(json_bytes)) ) return reqDeferred @implementer(IPolicyForHTTPS) class SydentPolicyForHTTPS: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def creatorForNetloc( self, hostname: bytes, port: int ) -> IOpenSSLClientConnectionCreator: return optionsForClientTLS( hostname.decode("ascii"), trustRoot=self.sydent.sslComponents.trustRoot, clientCertificate=self.sydent.sslComponents.myPrivateCertificate, ) sydent-2.5.1/sydent/http/httpserver.py000066400000000000000000000201151414516477000201020ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2018 New Vector Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING import twisted.internet.ssl from twisted.web.resource import Resource from twisted.web.server import Site from sydent.http.httpcommon import SizeLimitingRequest from sydent.http.servlets.authenticated_bind_threepid_servlet import ( AuthenticatedBindThreePidServlet, ) from sydent.http.servlets.authenticated_unbind_threepid_servlet import ( AuthenticatedUnbindThreePidServlet, ) if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class ClientApiHttpServer: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent root = Resource() matrix = Resource() identity = Resource() api = Resource() v1 = self.sydent.servlets.v1 v2 = self.sydent.servlets.v2 validate = Resource() validate_v2 = Resource() email = Resource() email_v2 = Resource() msisdn = Resource() msisdn_v2 = Resource() threepid_v1 = Resource() threepid_v2 = Resource() unbind = self.sydent.servlets.threepidUnbind pubkey = Resource() ephemeralPubkey = Resource() root.putChild(b"_matrix", matrix) matrix.putChild(b"identity", identity) identity.putChild(b"api", api) identity.putChild(b"v2", v2) api.putChild(b"v1", v1) validate.putChild(b"email", email) validate.putChild(b"msisdn", msisdn) validate_v2.putChild(b"email", email_v2) validate_v2.putChild(b"msisdn", msisdn_v2) v1.putChild(b"validate", validate) v1.putChild(b"lookup", self.sydent.servlets.lookup) v1.putChild(b"bulk_lookup", self.sydent.servlets.bulk_lookup) v1.putChild(b"pubkey", pubkey) pubkey.putChild(b"isvalid", self.sydent.servlets.pubkeyIsValid) pubkey.putChild(b"ed25519:0", self.sydent.servlets.pubkey_ed25519) pubkey.putChild(b"ephemeral", ephemeralPubkey) ephemeralPubkey.putChild( b"isvalid", self.sydent.servlets.ephemeralPubkeyIsValid ) threepid_v2.putChild( b"getValidated3pid", self.sydent.servlets.getValidated3pidV2 ) threepid_v2.putChild(b"bind", self.sydent.servlets.threepidBindV2) threepid_v2.putChild(b"unbind", unbind) threepid_v1.putChild(b"getValidated3pid", self.sydent.servlets.getValidated3pid) threepid_v1.putChild(b"unbind", unbind) if self.sydent.config.general.enable_v1_associations: threepid_v1.putChild(b"bind", self.sydent.servlets.threepidBind) v1.putChild(b"3pid", threepid_v1) email.putChild(b"requestToken", self.sydent.servlets.emailRequestCode) email.putChild(b"submitToken", self.sydent.servlets.emailValidate) email_v2.putChild(b"requestToken", self.sydent.servlets.emailRequestCodeV2) email_v2.putChild(b"submitToken", self.sydent.servlets.emailValidateV2) msisdn.putChild(b"requestToken", self.sydent.servlets.msisdnRequestCode) msisdn.putChild(b"submitToken", self.sydent.servlets.msisdnValidate) msisdn_v2.putChild(b"requestToken", self.sydent.servlets.msisdnRequestCodeV2) msisdn_v2.putChild(b"submitToken", self.sydent.servlets.msisdnValidateV2) v1.putChild(b"store-invite", self.sydent.servlets.storeInviteServlet) v1.putChild(b"sign-ed25519", self.sydent.servlets.blindlySignStuffServlet) # v2 # note v2 loses the /api so goes on 'identity' not 'api' identity.putChild(b"v2", v2) # v2 exclusive APIs v2.putChild(b"terms", self.sydent.servlets.termsServlet) account = self.sydent.servlets.accountServlet v2.putChild(b"account", account) account.putChild(b"register", self.sydent.servlets.registerServlet) account.putChild(b"logout", self.sydent.servlets.logoutServlet) # v2 versions of existing APIs v2.putChild(b"validate", validate_v2) v2.putChild(b"pubkey", pubkey) v2.putChild(b"3pid", threepid_v2) v2.putChild(b"store-invite", self.sydent.servlets.storeInviteServletV2) v2.putChild(b"sign-ed25519", self.sydent.servlets.blindlySignStuffServletV2) v2.putChild(b"lookup", self.sydent.servlets.lookup_v2) v2.putChild(b"hash_details", self.sydent.servlets.hash_details) self.factory = Site(root, SizeLimitingRequest) self.factory.displayTracebacks = False def setup(self) -> None: httpPort = self.sydent.config.http.client_port interface = self.sydent.config.http.client_bind_address logger.info("Starting Client API HTTP server on %s:%d", interface, httpPort) self.sydent.reactor.listenTCP( httpPort, self.factory, backlog=50, # taken from PosixReactorBase.listenTCP interface=interface, ) class InternalApiHttpServer: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def setup(self, interface: str, port: int) -> None: logger.info("Starting Internal API HTTP server on %s:%d", interface, port) root = Resource() matrix = Resource() root.putChild(b"_matrix", matrix) identity = Resource() matrix.putChild(b"identity", identity) internal = Resource() identity.putChild(b"internal", internal) authenticated_bind = AuthenticatedBindThreePidServlet(self.sydent) internal.putChild(b"bind", authenticated_bind) authenticated_unbind = AuthenticatedUnbindThreePidServlet(self.sydent) internal.putChild(b"unbind", authenticated_unbind) factory = Site(root) factory.displayTracebacks = False self.sydent.reactor.listenTCP( port, factory, backlog=50, # taken from PosixReactorBase.listenTCP interface=interface, ) class ReplicationHttpsServer: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent root = Resource() matrix = Resource() identity = Resource() root.putChild(b"_matrix", matrix) matrix.putChild(b"identity", identity) replicate = Resource() replV1 = Resource() identity.putChild(b"replicate", replicate) replicate.putChild(b"v1", replV1) replV1.putChild(b"push", self.sydent.servlets.replicationPush) self.factory = Site(root) self.factory.displayTracebacks = False def setup(self) -> None: httpPort = self.sydent.config.http.replication_port interface = self.sydent.config.http.replication_bind_address if self.sydent.sslComponents.myPrivateCertificate: # We will already have logged a warn if this is absent, so don't do it again cert = self.sydent.sslComponents.myPrivateCertificate certOptions = twisted.internet.ssl.CertificateOptions( privateKey=cert.privateKey.original, certificate=cert.original, trustRoot=self.sydent.sslComponents.trustRoot, ) logger.info("Loaded server private key and certificate!") logger.info( "Starting Replication HTTPS server on %s:%d", interface, httpPort ) self.sydent.reactor.listenSSL( httpPort, self.factory, certOptions, backlog=50, # taken from PosixReactorBase.listenTCP interface=interface, ) sydent-2.5.1/sydent/http/matrixfederationagent.py000066400000000000000000000430351414516477000222660ustar00rootroot00000000000000# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import random import time from typing import Any, Callable, Dict, Generator, Optional, Tuple import attr from netaddr import IPAddress from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet.interfaces import ( IProtocol, IProtocolFactory, IReactorTime, IStreamClientEndpoint, ) from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent from twisted.web.http import stringToDatetime from twisted.web.http_headers import Headers from twisted.web.iweb import ( IAgent, IAgentEndpointFactory, IBodyProducer, IPolicyForHTTPS, IResponse, ) from zope.interface import implementer from sydent.http.federation_tls_options import ClientTLSOptionsFactory from sydent.http.httpcommon import read_body_with_max_size from sydent.http.srvresolver import SrvResolver, pick_server_from_list from sydent.util import json_decoder from sydent.util.ttlcache import TTLCache # period to cache .well-known results for by default WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600 # jitter to add to the .well-known default cache ttl WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER = 10 * 60 # period to cache failure to fetch .well-known for WELL_KNOWN_INVALID_CACHE_PERIOD = 1 * 3600 # cap for .well-known cache period WELL_KNOWN_MAX_CACHE_PERIOD = 48 * 3600 # The maximum size (in bytes) to allow a well-known file to be. WELL_KNOWN_MAX_SIZE = 50 * 1024 # 50 KiB logger = logging.getLogger(__name__) well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known") @implementer(IAgent) class MatrixFederationAgent: """An Agent-like thing which provides a `request` method which will look up a matrix server and send an HTTP request to it. Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.) :param reactor: twisted reactor to use for underlying requests :param tls_client_options_factory: Factory to use for fetching client tls options, or none to disable TLS. :param _well_known_tls_policy: TLS policy to use for fetching .well-known files. None to use a default (browser-like) implementation. :param _well_known_cache: TTLCache impl for storing cached well-known lookups. Omit to use a default implementation. """ def __init__( self, # This reactor should also be IReactorTCP and IReactorPluggableNameResolver # because it eventually makes its way to HostnameEndpoint.__init__. # But that's not easy to express with an annotation. We use the # `seconds` attribute below, so mark this as IReactorTime for now. reactor: IReactorTime, tls_client_options_factory: Optional[ClientTLSOptionsFactory], _well_known_tls_policy: Optional[IPolicyForHTTPS] = None, _srv_resolver: Optional[SrvResolver] = None, _well_known_cache: TTLCache[bytes, Optional[bytes]] = well_known_cache, ) -> None: self._reactor = reactor self._tls_client_options_factory = tls_client_options_factory if _srv_resolver is None: _srv_resolver = SrvResolver() self._srv_resolver = _srv_resolver self._pool = HTTPConnectionPool(reactor) self._pool.retryAutomatically = False self._pool.maxPersistentPerHost = 5 self._pool.cachedConnectionTimeout = 2 * 60 if _well_known_tls_policy is not None: # the param is called 'contextFactory', but actually passing a # contextfactory is deprecated, and it expects an IPolicyForHTTPS. _well_known_agent = Agent( self._reactor, pool=self._pool, contextFactory=_well_known_tls_policy ) else: _well_known_agent = Agent(self._reactor, pool=self._pool) self._well_known_agent = RedirectAgent(_well_known_agent) # our cache of .well-known lookup results, mapping from server name # to delegated name. The values can be: # `bytes`: a valid server-name # `None`: there is no (valid) .well-known here self._well_known_cache = _well_known_cache @defer.inlineCallbacks def request( self, method: bytes, uri: bytes, headers: Optional["Headers"] = None, bodyProducer: Optional["IBodyProducer"] = None, ) -> Generator["defer.Deferred[Any]", Any, IResponse]: """ :param method: HTTP method (GET/POST/etc). :param uri: Absolute URI to be retrieved. :param headers: HTTP headers to send with the request, or None to send no extra headers. :param bodyProducer: An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or None if the request is to have no body. :returns a deferred that fires when the header of the response has been received (regardless of the response status code). Fails if there is any problem which prevents that response from being received (including problems that prevent the request from being sent). """ parsed_uri = URI.fromBytes(uri, defaultPort=-1) routing: _RoutingResult routing = yield defer.ensureDeferred(self._route_matrix_uri(parsed_uri)) # set up the TLS connection params # # XXX disabling TLS is really only supported here for the benefit of the # unit tests. We should make the UTs cope with TLS rather than having to make # the code support the unit tests. if self._tls_client_options_factory is None: tls_options = None else: tls_options = self._tls_client_options_factory.get_options( routing.tls_server_name.decode("ascii") ) # make sure that the Host header is set correctly if headers is None: headers = Headers() else: # Type safety: Headers.copy doesn't have a return type annotated, # and I don't want to stub web.http_headers. Could use stubgen? It's # a pretty simple file. headers = headers.copy() # type: ignore[no-untyped-call] assert headers is not None if not headers.hasHeader(b"host"): headers.addRawHeader(b"host", routing.host_header) @implementer(IAgentEndpointFactory) class EndpointFactory: @staticmethod def endpointForURI(_uri: URI) -> IStreamClientEndpoint: ep: IStreamClientEndpoint = LoggingHostnameEndpoint( self._reactor, routing.target_host, routing.target_port, ) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool) res: IResponse res = yield agent.request(method, uri, headers, bodyProducer) return res async def _route_matrix_uri( self, parsed_uri: "URI", lookup_well_known: bool = True ) -> "_RoutingResult": """Helper for `request`: determine the routing for a Matrix URI :param parsed_uri: uri to route. Note that it should be parsed with URI.fromBytes(uri, defaultPort=-1) to set the `port` to -1 if there is no explicit port given. :param lookup_well_known: True if we should look up the .well-known file if there is no SRV record. :returns a routing result. """ # check for an IP literal try: ip_address = IPAddress(parsed_uri.host.decode("ascii")) except Exception: # not an IP address ip_address = None if ip_address: port = parsed_uri.port if port == -1: port = 8448 return _RoutingResult( host_header=parsed_uri.netloc, tls_server_name=parsed_uri.host, target_host=parsed_uri.host, target_port=port, ) if parsed_uri.port != -1: # there is an explicit port return _RoutingResult( host_header=parsed_uri.netloc, tls_server_name=parsed_uri.host, target_host=parsed_uri.host, target_port=parsed_uri.port, ) if lookup_well_known: # try a .well-known lookup well_known_server = await self._get_well_known(parsed_uri.host) if well_known_server: # if we found a .well-known, start again, but don't do another # .well-known lookup. # parse the server name in the .well-known response into host/port. # (This code is lifted from twisted.web.client.URI.fromBytes). if b":" in well_known_server: well_known_host, well_known_port_raw = well_known_server.rsplit( b":", 1 ) try: well_known_port = int(well_known_port_raw) except ValueError: # the part after the colon could not be parsed as an int # - we assume it is an IPv6 literal with no port (the closing # ']' stops it being parsed as an int) well_known_host, well_known_port = well_known_server, -1 else: well_known_host, well_known_port = well_known_server, -1 new_uri = URI( scheme=parsed_uri.scheme, netloc=well_known_server, host=well_known_host, port=well_known_port, path=parsed_uri.path, params=parsed_uri.params, query=parsed_uri.query, fragment=parsed_uri.fragment, ) res = await self._route_matrix_uri(new_uri, lookup_well_known=False) return res # try a SRV lookup service_name = b"_matrix._tcp.%s" % (parsed_uri.host,) server_list = await self._srv_resolver.resolve_service(service_name) if not server_list: target_host = parsed_uri.host port = 8448 logger.debug( "No SRV record for %s, using %s:%i", parsed_uri.host.decode("ascii"), target_host.decode("ascii"), port, ) else: target_host, port = pick_server_from_list(server_list) logger.debug( "Picked %s:%i from SRV records for %s", target_host.decode("ascii"), port, parsed_uri.host.decode("ascii"), ) return _RoutingResult( host_header=parsed_uri.netloc, tls_server_name=parsed_uri.host, target_host=target_host, target_port=port, ) async def _get_well_known(self, server_name: bytes) -> Optional[bytes]: """Attempt to fetch and parse a .well-known file for the given server :param server_name: Name of the server, from the requested url. :returns either the new server name, from the .well-known, or None if there was no .well-known file. """ try: result = self._well_known_cache[server_name] except KeyError: # TODO: should we linearise so that we don't end up doing two .well-known # requests for the same server in parallel? result, cache_period = await self._do_get_well_known(server_name) if cache_period > 0: self._well_known_cache.set(server_name, result, cache_period) return result async def _do_get_well_known( self, server_name: bytes ) -> Tuple[Optional[bytes], float]: """Actually fetch and parse a .well-known, without checking the cache :param server_name: Name of the server, from the requested url :returns a tuple of (result, cache period), where result is one of: - the new server name from the .well-known (as a `bytes`) - None if there was no .well-known file. - INVALID_WELL_KNOWN if the .well-known was invalid """ uri = b"https://%s/.well-known/matrix/server" % (server_name,) uri_str = uri.decode("ascii") logger.info("Fetching %s", uri_str) cache_period: Optional[float] try: response = await self._well_known_agent.request(b"GET", uri) body = await read_body_with_max_size(response, WELL_KNOWN_MAX_SIZE) if response.code != 200: raise Exception("Non-200 response %s" % (response.code,)) parsed_body = json_decoder.decode(body.decode("utf-8")) logger.info("Response from .well-known: %s", parsed_body) if not isinstance(parsed_body, dict): raise Exception("not a dict") if "m.server" not in parsed_body: raise Exception("Missing key 'm.server'") except Exception as e: logger.info("Error fetching %s: %s", uri_str, e) # add some randomness to the TTL to avoid a stampeding herd every hour # after startup cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER) return (None, cache_period) result = parsed_body["m.server"].encode("ascii") cache_period = _cache_period_from_headers( response.headers, time_now=self._reactor.seconds, ) if cache_period is None: cache_period = WELL_KNOWN_DEFAULT_CACHE_PERIOD # add some randomness to the TTL to avoid a stampeding herd every 24 hours # after startup cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER) else: cache_period = min(cache_period, WELL_KNOWN_MAX_CACHE_PERIOD) return (result, cache_period) @implementer(IStreamClientEndpoint) class LoggingHostnameEndpoint: """A wrapper for HostnameEndpint which logs when it connects""" def __init__( self, reactor: IReactorTime, host: bytes, port: int, *args: Any, **kwargs: Any ): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) logger.info("Endpoint created with %s:%d", host, port) def connect( self, protocol_factory: IProtocolFactory ) -> "defer.Deferred[IProtocol]": logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port) return self.ep.connect(protocol_factory) def _cache_period_from_headers( headers: Headers, time_now: Callable[[], float] = time.time ) -> Optional[float]: cache_controls = _parse_cache_control(headers) if b"no-store" in cache_controls: return 0 max_age = cache_controls.get(b"max-age") if max_age is not None: try: return int(max_age) except ValueError: pass expires = headers.getRawHeaders(b"expires") if expires is not None: try: expires_date = stringToDatetime(expires[-1]) return expires_date - time_now() except ValueError: # RFC7234 says 'A cache recipient MUST interpret invalid date formats, # especially the value "0", as representing a time in the past (i.e., # "already expired"). return 0 return None def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]: cache_controls: Dict[bytes, Optional[bytes]] = {} for hdr in headers.getRawHeaders(b"cache-control", []): for directive in hdr.split(b","): splits = [x.strip() for x in directive.split(b"=", 1)] k = splits[0].lower() v = splits[1] if len(splits) > 1 else None cache_controls[k] = v return cache_controls @attr.s(frozen=True, slots=True, auto_attribs=True) class _RoutingResult: """The result returned by `_route_matrix_uri`. Contains the parameters needed to direct a federation connection to a particular server. Where a SRV record points to several servers, this object contains a single server chosen from the list. """ host_header: bytes """ The value we should assign to the Host header (host:port from the matrix URI, or .well-known). """ tls_server_name: bytes """ The server name we should set in the SNI (typically host, without port, from the matrix URI or .well-known) """ target_host: bytes """ The hostname (or IP literal) we should route the TCP connection to (the target of the SRV record, or the hostname from the URL/.well-known) """ target_port: int """ The port we should route the TCP connection to (the target of the SRV record, or the port from the URL/.well-known, or 8448) """ sydent-2.5.1/sydent/http/servlets/000077500000000000000000000000001414516477000171725ustar00rootroot00000000000000sydent-2.5.1/sydent/http/servlets/__init__.py000066400000000000000000000211341414516477000213040ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import functools import json import logging from typing import Any, Awaitable, Callable, Dict, Iterable, TypeVar from twisted.internet import defer from twisted.web import server from twisted.web.resource import Resource from twisted.web.server import Request from sydent.types import JsonDict from sydent.util import json_decoder logger = logging.getLogger(__name__) class MatrixRestError(Exception): """ Handled by the jsonwrap wrapper. Any servlets that don't use this wrapper should catch this exception themselves. """ def __init__(self, httpStatus: int, errcode: str, error: str): super(Exception, self).__init__(error) self.httpStatus = httpStatus self.errcode = errcode self.error = error def get_args( request: Request, args: Iterable[str], required: bool = True ) -> Dict[str, Any]: """ Helper function to get arguments for an HTTP request. Currently takes args from the top level keys of a json object or www-form-urlencoded for backwards compatibility on v1 endpoints only. ⚠️ BEWARE ⚠. If a v1 request provides its args in urlencoded form (either in a POST body or as URL query parameters), then we'll return `Dict[str, str]`. The caller may need to interpret these strings as e.g. an `int`, `bool`, etc. Arguments given as a json body are processed with `json.JSONDecoder.decode`, and so are automatically deserialised to a Python type. The caller should still validate that these have the correct type! :param request: The request received by the servlet. :param args: The args to look for in the request's parameters. :param required: Whether to raise a MatrixRestError with 400 M_MISSING_PARAMS if an argument is not found. :raises: MatrixRestError if required is True and a given parameter was not found in the request's query parameters. :raises: MatrixRestError if we the request body contains bad JSON. :raises: MatrixRestError if arguments are given in www-form-urlencodedquery form, and some argument name or value is not a valid UTF-8-encoded string. :return: A dict containing the requested args and their values. String values are of type unicode. """ assert request.path is not None v1_path = request.path.startswith(b"/_matrix/identity/api/v1") request_args = None # for v1 paths, only look for json args if content type is json if request.method in (b"POST", b"PUT") and ( not v1_path or ( request.requestHeaders.hasHeader("Content-Type") # type safety: getRawHeaders() will return a nonempty list because # the hasHeader call has returned True. and request.requestHeaders.getRawHeaders("Content-Type")[0].startswith( # type: ignore[index] "application/json" ) ) ): try: # json.loads doesn't allow bytes in Python 3.5 request_args = json_decoder.decode(request.content.read().decode("UTF-8")) except ValueError: raise MatrixRestError(400, "M_BAD_JSON", "Malformed JSON") # If we didn't get anything from that, and it's a v1 api path, try the request args # (element-web's usage of the ed25519 sign servlet currently involves # sending the params in the query string with a json body of 'null') if request_args is None and (v1_path or request.method == b"GET"): request_args_bytes = copy.copy(request.args) # Twisted supplies everything as an array because it's valid to # supply the same params multiple times with www-form-urlencoded # params. This make it incompatible with the json object though, # so we need to convert one of them. Since this is the # backwards-compat option, we convert this one. request_args = {} for k, v in request_args_bytes.items(): if isinstance(v, list) and len(v) == 1: try: request_args[k.decode("UTF-8")] = v[0].decode("UTF-8") except UnicodeDecodeError: # Get a version of the key that has non-UTF-8 characters replaced by # their \xNN escape sequence so it doesn't raise another exception. safe_k = k.decode("UTF-8", errors="backslashreplace") raise MatrixRestError( 400, "M_INVALID_PARAM", "Parameter %s and its value must be valid UTF-8" % safe_k, ) elif request_args is None: request_args = {} if required: # Check for any missing arguments missing = [] for a in args: if a not in request_args: missing.append(a) if len(missing) > 0: request.setResponseCode(400) msg = "Missing parameters: " + (",".join(missing)) raise MatrixRestError(400, "M_MISSING_PARAMS", msg) return request_args Res = TypeVar("Res", bound=Resource) def jsonwrap(f: Callable[[Res, Request], JsonDict]) -> Callable[[Res, Request], bytes]: @functools.wraps(f) def inner(self: Res, request: Request) -> bytes: """ Runs a web handler function with the given request and parameters, then converts its result into JSON and returns it. If an error happens, also sets the HTTP response code. :param self: The current object. :param request: The request to process. :return: The JSON payload to send as a response to the request. """ try: request.setHeader("Content-Type", "application/json") return dict_to_json_bytes(f(self, request)) except MatrixRestError as e: request.setResponseCode(e.httpStatus) return dict_to_json_bytes({"errcode": e.errcode, "error": e.error}) except Exception: logger.exception("Exception processing request") request.setHeader("Content-Type", "application/json") request.setResponseCode(500) return dict_to_json_bytes( { "errcode": "M_UNKNOWN", "error": "Internal Server Error", } ) return inner AsyncRenderer = Callable[[Res, Request], Awaitable[JsonDict]] def asyncjsonwrap(f: AsyncRenderer[Res]) -> Callable[[Res, Request], object]: async def render(f: AsyncRenderer[Res], self: Res, request: Request) -> None: request.setHeader("Content-Type", "application/json") try: result = await f(self, request) request.write(dict_to_json_bytes(result)) except MatrixRestError as e: request.setResponseCode(e.httpStatus) request.write(dict_to_json_bytes({"errcode": e.errcode, "error": e.error})) except Exception: logger.exception("Request processing failed") request.setResponseCode(500) request.write( dict_to_json_bytes( {"errcode": "M_UNKNOWN", "error": "Internal Server Error"} ) ) request.finish() @functools.wraps(f) def inner(self: Res, request: Request) -> object: """ Runs an asynchronous web handler function with the given arguments. :param self: The servelet instance :param request: The request that `f` will serve. :return: An opaque object to tell the servlet that the response isn't ready yet and will come later. """ defer.ensureDeferred(render(f, self, request)) return server.NOT_DONE_YET return inner def send_cors(request: Request) -> None: request.setHeader("Access-Control-Allow-Origin", "*") request.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") request.setHeader("Access-Control-Allow-Headers", "*") def dict_to_json_bytes(content: JsonDict) -> bytes: """ Converts a dict into JSON and encodes it to bytes. :return: The JSON bytes. """ return json.dumps(content).encode("UTF-8") sydent-2.5.1/sydent/http/servlets/accountservlet.py000066400000000000000000000027141414516477000226110ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.auth import authV2 from sydent.http.servlets import jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class AccountServlet(Resource): isLeaf = False def __init__(self, syd: "Sydent") -> None: Resource.__init__(self) self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: """ Return information about the user's account (essentially just a 'who am i') """ send_cors(request) account = authV2(self.sydent, request) return { "user_id": account.userId, } def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/authenticated_bind_threepid_servlet.py000066400000000000000000000030431414516477000270120ustar00rootroot00000000000000# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class AuthenticatedBindThreePidServlet(Resource): """A servlet which allows a caller to bind any 3pid they want to an mxid It is assumed that authentication happens out of band """ def __init__(self, sydent: "Sydent") -> None: Resource.__init__(self) self.sydent = sydent @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) args = get_args(request, ("medium", "address", "mxid")) return self.sydent.threepidBinder.addBinding( args["medium"], args["address"], args["mxid"], ) def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/authenticated_unbind_threepid_servlet.py000066400000000000000000000031351414516477000273570ustar00rootroot00000000000000# Copyright 2020 Dirk Klimpel # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class AuthenticatedUnbindThreePidServlet(Resource): """A servlet which allows a caller to unbind any 3pid they want from an mxid It is assumed that authentication happens out of band """ def __init__(self, sydent: "Sydent") -> None: Resource.__init__(self) self.sydent = sydent @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) args = get_args(request, ("medium", "address", "mxid")) threepid = {"medium": args["medium"], "address": args["address"]} self.sydent.threepidBinder.removeBinding( threepid, args["mxid"], ) return {} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/blindlysignstuffservlet.py000066400000000000000000000047531414516477000245500ustar00rootroot00000000000000# Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING import signedjson.key import signedjson.sign from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.invite_tokens import JoinTokenStore from sydent.http.auth import authV2 from sydent.http.servlets import MatrixRestError, get_args, jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class BlindlySignStuffServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.server_name = syd.config.general.server_name self.tokenStore = JoinTokenStore(syd) self.require_auth = require_auth @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) args = get_args(request, ("private_key", "token", "mxid")) private_key_base64 = args["private_key"] token = args["token"] mxid = args["mxid"] sender = self.tokenStore.getSenderForToken(token) if sender is None: raise MatrixRestError(404, "M_UNRECOGNIZED", "Didn't recognize token") to_sign = { "mxid": mxid, "sender": sender, "token": token, } try: private_key = signedjson.key.decode_signing_key_base64( "ed25519", "0", private_key_base64 ) signed: JsonDict = signedjson.sign.sign_json( to_sign, self.server_name, private_key ) except Exception: logger.exception("signing failed") raise MatrixRestError(500, "M_UNKNOWN", "Internal Server Error") return signed def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/bulklookupservlet.py000066400000000000000000000042161414516477000233430ustar00rootroot00000000000000# Copyright 2017 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.threepid_associations import GlobalAssociationStore from sydent.http.servlets import MatrixRestError, get_args, jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class BulkLookupServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_POST(self, request: Request) -> JsonDict: """ Bulk-lookup for threepids. Params: 'threepids': list of threepids, each of which is a list of medium, address Returns: Object with key 'threepids', which is a list of results where each result is a 3 item list of medium, address, mxid Note that results are not streamed to the client. Threepids for which no mapping is found are omitted. """ send_cors(request) args = get_args(request, ("threepids",)) threepids = args["threepids"] if not isinstance(threepids, list): raise MatrixRestError(400, "M_INVALID_PARAM", "threepids must be a list") logger.info("Bulk lookup of %d threepids", len(threepids)) globalAssocStore = GlobalAssociationStore(self.sydent) results = globalAssocStore.getMxids(threepids) return {"threepids": results} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/emailservlet.py000066400000000000000000000177011414516477000222460ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING, Optional from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.auth import authV2 from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util.emailutils import EmailAddressException, EmailSendException from sydent.util.stringutils import MAX_EMAIL_ADDRESS_LENGTH, is_valid_client_secret from sydent.validators import ( IncorrectClientSecretException, IncorrectSessionTokenException, InvalidSessionIdException, SessionExpiredException, ) if TYPE_CHECKING: from sydent.sydent import Sydent class EmailRequestCodeServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.require_auth = require_auth @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) args = get_args(request, ("email", "client_secret", "send_attempt")) email = args["email"] clientSecret = args["client_secret"] try: # if we got this via the v1 API in a querystring or urlencoded body, # then the values in args will be a string. So check that # send_attempt is an int. # # NB: We don't check if we're processing a url-encoded v1 request. # This means we accept string representations of integers for # `send_attempt` in v2 requests, and in v1 requests that supply a # JSON body. This is contrary to the spec and leaves me with a dirty # feeling I can't quite shake off. # # Where's Raymond Hettinger when you need him? (THUMP) There must be # a better way! sendAttempt = int(args["send_attempt"]) except (TypeError, ValueError): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": f"send_attempt should be an integer (got {args['send_attempt']}", } if not is_valid_client_secret(clientSecret): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } if not (0 < len(email) <= MAX_EMAIL_ADDRESS_LENGTH): request.setResponseCode(400) return {"errcode": "M_INVALID_PARAM", "error": "Invalid email provided"} ipaddress = self.sydent.ip_from_request(request) brand = self.sydent.brand_from_request(request) nextLink: Optional[str] = None if "next_link" in args and not args["next_link"].startswith("file:///"): nextLink = args["next_link"] try: sid = self.sydent.validators.email.requestToken( email, clientSecret, sendAttempt, nextLink, ipaddress=ipaddress, brand=brand, ) resp = {"sid": str(sid)} except EmailAddressException: request.setResponseCode(400) resp = {"errcode": "M_INVALID_EMAIL", "error": "Invalid email address"} except EmailSendException: request.setResponseCode(500) resp = {"errcode": "M_EMAIL_SEND_ERROR", "error": "Failed to send email"} return resp def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" class EmailValidateCodeServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.require_auth = require_auth def render_GET(self, request: Request) -> bytes: args = get_args(request, ("nextLink",), required=False) resp = None try: resp = self.do_validate_request(request) except Exception: pass if resp and "success" in resp and resp["success"]: msg = "Verification successful! Please return to your Matrix client to continue." if "nextLink" in args: next_link = args["nextLink"] if not next_link.startswith("file:///"): request.setResponseCode(302) request.setHeader("Location", next_link) else: msg = "Verification failed: you may need to request another verification email" brand = self.sydent.brand_from_request(request) # self.sydent.config.http.verify_response_template is deprecated if self.sydent.config.http.verify_response_template is None: templateFile = self.sydent.get_branded_template( brand, "verify_response_template.html", ) else: templateFile = self.sydent.config.http.verify_response_template request.setHeader("Content-Type", "text/html") res = open(templateFile).read() % {"message": msg} return res.encode("UTF-8") @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) return self.do_validate_request(request) def do_validate_request(self, request: Request) -> JsonDict: """ Extracts information about a validation session from the request and attempts to validate that session. :param request: The request to extract information about the session from. :return: A dict with a "success" key which value indicates whether the validation succeeded. If the validation failed, this dict also includes a "errcode" and a "error" keys which include information about the failure. """ args = get_args(request, ("token", "sid", "client_secret")) sid = args["sid"] tokenString = args["token"] clientSecret = args["client_secret"] if not is_valid_client_secret(clientSecret): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } try: return self.sydent.validators.email.validateSessionWithToken( sid, clientSecret, tokenString ) except IncorrectClientSecretException: return { "success": False, "errcode": "M_INVALID_PARAM", "error": "Client secret does not match the one given when requesting the token", } except SessionExpiredException: return { "success": False, "errcode": "M_SESSION_EXPIRED", "error": "This validation session has expired: call requestToken again", } except InvalidSessionIdException: return { "success": False, "errcode": "M_INVALID_PARAM", "error": "The token doesn't match", } except IncorrectSessionTokenException: return { "success": False, "errcode": "M_NO_VALID_SESSION", "error": "No session could be found with this sid", } def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/getvalidated3pidservlet.py000066400000000000000000000057711414516477000244000ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.valsession import ThreePidValSessionStore from sydent.http.auth import authV2 from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util.stringutils import is_valid_client_secret from sydent.validators import ( IncorrectClientSecretException, InvalidSessionIdException, SessionExpiredException, SessionNotValidatedException, ) if TYPE_CHECKING: from sydent.sydent import Sydent class GetValidated3pidServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.require_auth = require_auth @jsonwrap def render_GET(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) args = get_args(request, ("sid", "client_secret")) sid = args["sid"] clientSecret = args["client_secret"] if not is_valid_client_secret(clientSecret): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } valSessionStore = ThreePidValSessionStore(self.sydent) noMatchError = { "errcode": "M_NO_VALID_SESSION", "error": "No valid session was found matching that sid and client secret", } try: s = valSessionStore.getValidatedSession(sid, clientSecret) except (IncorrectClientSecretException, InvalidSessionIdException): request.setResponseCode(404) return noMatchError except SessionExpiredException: request.setResponseCode(400) return { "errcode": "M_SESSION_EXPIRED", "error": "This validation session has expired: call requestToken again", } except SessionNotValidatedException: request.setResponseCode(400) return { "errcode": "M_SESSION_NOT_VALIDATED", "error": "This validation session has not yet been completed", } return {"medium": s.medium, "address": s.address, "validated_at": s.mtime} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/hashdetailsservlet.py000066400000000000000000000037431414516477000234510ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.auth import authV2 from sydent.http.servlets import jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class HashDetailsServlet(Resource): isLeaf = True known_algorithms = ["sha256", "none"] def __init__(self, syd: "Sydent", lookup_pepper: str) -> None: self.sydent = syd self.lookup_pepper = lookup_pepper @jsonwrap def render_GET(self, request: Request) -> JsonDict: """ Return the hashing algorithms and pepper that this IS supports. The pepper included in the response is stored in the database, or otherwise generated. Returns: An object containing an array of hashing algorithms the server supports, and a `lookup_pepper` field, which is a server-defined value that the client should include in the 3PID information before hashing. """ send_cors(request) authV2(self.sydent, request) return { "algorithms": self.known_algorithms, "lookup_pepper": self.lookup_pepper, } def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/logoutservlet.py000066400000000000000000000032611414516477000224640ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.accounts import AccountStore from sydent.http.auth import authV2, tokenFromRequest from sydent.http.servlets import MatrixRestError, jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class LogoutServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_POST(self, request: Request) -> JsonDict: """ Invalidate the given access token """ send_cors(request) authV2(self.sydent, request, False) token = tokenFromRequest(request) if token is None: raise MatrixRestError(400, "M_MISSING_PARAMS", "Missing token") accountStore = AccountStore(self.sydent) accountStore.delToken(token) return {} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/lookupservlet.py000066400000000000000000000066231414516477000224710ustar00rootroot00000000000000# Copyright 2014,2017 OpenMarket Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING import signedjson.sign from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.threepid_associations import GlobalAssociationStore from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util import json_decoder if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class LookupServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: """ Look up an individual threepid. ** DEPRECATED ** Params: 'medium': the medium of the threepid 'address': the address of the threepid Returns: A signed association if the threepid has a corresponding mxid, otherwise the empty object. """ send_cors(request) args = get_args(request, ("medium", "address")) medium = args["medium"] address = args["address"] globalAssocStore = GlobalAssociationStore(self.sydent) sgassoc_raw = globalAssocStore.signedAssociationStringForThreepid( medium, address ) if not sgassoc_raw: return {} # TODO validate this really is a dict sgassoc: JsonDict = json_decoder.decode(sgassoc_raw) if self.sydent.config.general.server_name not in sgassoc["signatures"]: # We have not yet worked out what the proper trust model should be. # # Maybe clients implicitly trust a server they talk to (and so we # should sign every assoc we return as ourselves, so they can # verify this). # # Maybe clients really want to know what server did the original # verification, and want to only know exactly who signed the assoc. # # Until we work out what we should do, sign all assocs we return as # ourself. This is vaguely ok because there actually is only one # identity server, but it happens to have two names (matrix.org and # vector.im), and so we're not really lying too much. # # We do this when we return assocs, not when we receive them over # replication, so that we can undo this decision in the future if # we wish, without having destroyed the raw underlying data. sgassoc = signedjson.sign.sign_json( sgassoc, self.sydent.config.general.server_name, self.sydent.keyring.ed25519, ) return sgassoc def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/lookupv2servlet.py000066400000000000000000000131271414516477000227360ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.threepid_associations import GlobalAssociationStore from sydent.http.auth import authV2 from sydent.http.servlets import get_args, jsonwrap, send_cors from sydent.http.servlets.hashdetailsservlet import HashDetailsServlet from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class LookupV2Servlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", lookup_pepper: str) -> None: self.sydent = syd self.globalAssociationStore = GlobalAssociationStore(self.sydent) self.lookup_pepper = lookup_pepper @jsonwrap def render_POST(self, request: Request) -> JsonDict: """ Perform lookups with potentially hashed 3PID details. Depending on our response to /hash_details, the client will choose a hash algorithm and pepper, hash the 3PIDs it wants to lookup, and send them to us, along with the algorithm and pepper it used. We first check this algorithm/pepper combo matches what we expect, then compare the 3PID details to what we have in the database. Params: A JSON object containing the following keys: * 'addresses': List of hashed/plaintext (depending on the algorithm) 3PID addresses and mediums. * 'algorithm': The algorithm the client has used to process the 3PIDs. * 'pepper': The pepper the client has attached to the 3PIDs. Returns: Object with key 'mappings', which is a dictionary of results where each result is a key/value pair of what the client sent, and the matching Matrix User ID that claims to own that 3PID. User IDs for which no mapping is found are omitted. """ send_cors(request) authV2(self.sydent, request) args = get_args(request, ("addresses", "algorithm", "pepper")) addresses = args["addresses"] if not isinstance(addresses, list): request.setResponseCode(400) return {"errcode": "M_INVALID_PARAM", "error": "addresses must be a list"} algorithm = str(args["algorithm"]) if algorithm not in HashDetailsServlet.known_algorithms: request.setResponseCode(400) return {"errcode": "M_INVALID_PARAM", "error": "algorithm is not supported"} # Ensure address count is under the configured limit limit = self.sydent.config.general.address_lookup_limit if len(addresses) > limit: request.setResponseCode(400) return { "errcode": "M_TOO_LARGE", "error": "More than the maximum amount of " "addresses provided", } pepper = str(args["pepper"]) if pepper != self.lookup_pepper: request.setResponseCode(400) return { "errcode": "M_INVALID_PEPPER", "error": "pepper does not match '%s'" % (self.lookup_pepper,), "algorithm": algorithm, "lookup_pepper": self.lookup_pepper, } logger.info( "Lookup of %d threepid(s) with algorithm %s", len(addresses), algorithm ) if algorithm == "none": # Lookup without hashing medium_address_tuples = [] for address_and_medium in addresses: # Parse medium, address components address_medium_split = address_and_medium.split() # Forbid addresses that contain a space if len(address_medium_split) != 2: request.setResponseCode(400) return { "errcode": "M_UNKNOWN", "error": 'Invalid "address medium" pair: "%s"' % address_and_medium, } # Get the mxid for the address/medium combo if known address, medium = address_medium_split medium_address_tuples.append((medium, address)) # Lookup the mxids medium_address_mxid_tuples = self.globalAssociationStore.getMxids( medium_address_tuples ) # Return a dictionary of lookup_string: mxid values return { "mappings": { "%s %s" % (x[1], x[0]): x[2] for x in medium_address_mxid_tuples } } elif algorithm == "sha256": # Lookup using SHA256 with URL-safe base64 encoding mappings = self.globalAssociationStore.retrieveMxidsForHashes(addresses) return {"mappings": mappings} request.setResponseCode(400) return {"errcode": "M_INVALID_PARAM", "error": "algorithm is not supported"} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/msisdnservlet.py000066400000000000000000000203121414516477000224440ustar00rootroot00000000000000# Copyright 2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING import phonenumbers from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.auth import authV2 from sydent.http.servlets import asyncjsonwrap, get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util.stringutils import is_valid_client_secret from sydent.validators import ( DestinationRejectedException, IncorrectClientSecretException, IncorrectSessionTokenException, InvalidSessionIdException, SessionExpiredException, ) if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class MsisdnRequestCodeServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.require_auth = require_auth @asyncjsonwrap async def render_POST(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) args = get_args( request, ("phone_number", "country", "client_secret", "send_attempt") ) raw_phone_number = args["phone_number"] country = args["country"] try: # See the comment handling `send_attempt` in emailservlet.py for # more context. sendAttempt = int(args["send_attempt"]) except (TypeError, ValueError): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": f"send_attempt should be an integer (got {args['send_attempt']}", } clientSecret = args["client_secret"] if not is_valid_client_secret(clientSecret): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } try: phone_number_object = phonenumbers.parse(raw_phone_number, country) except Exception as e: logger.warning("Invalid phone number given: %r", e) request.setResponseCode(400) return { "errcode": "M_INVALID_PHONE_NUMBER", "error": "Invalid phone number", } msisdn = phonenumbers.format_number( phone_number_object, phonenumbers.PhoneNumberFormat.E164 )[1:] # International formatted number. The same as an E164 but with spaces # in appropriate places to make it nicer for the humans. intl_fmt = phonenumbers.format_number( phone_number_object, phonenumbers.PhoneNumberFormat.INTERNATIONAL ) brand = self.sydent.brand_from_request(request) try: sid = await self.sydent.validators.msisdn.requestToken( phone_number_object, clientSecret, sendAttempt, brand ) resp = { "success": True, "sid": str(sid), "msisdn": msisdn, "intl_fmt": intl_fmt, } except DestinationRejectedException: logger.warning("Destination rejected for number: %s", msisdn) request.setResponseCode(400) resp = { "errcode": "M_DESTINATION_REJECTED", "error": "Phone numbers in this country are not currently supported", } except Exception: logger.exception("Exception sending SMS") request.setResponseCode(500) resp = {"errcode": "M_UNKNOWN", "error": "Internal Server Error"} return resp def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" class MsisdnValidateCodeServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.require_auth = require_auth def render_GET(self, request: Request) -> str: send_cors(request) args = get_args(request, ("token", "sid", "client_secret")) resp = self.do_validate_request(request) if "success" in resp and resp["success"]: msg = "Verification successful! Please return to your Matrix client to continue." if "next_link" in args: next_link = args["next_link"] request.setResponseCode(302) request.setHeader("Location", next_link) else: request.setResponseCode(400) msg = ( "Verification failed: you may need to request another verification text" ) brand = self.sydent.brand_from_request(request) # self.sydent.config.http.verify_response_template is deprecated if self.sydent.config.http.verify_response_template is None: templateFile = self.sydent.get_branded_template( brand, "verify_response_template.html", ) else: templateFile = self.sydent.config.http.verify_response_template request.setHeader("Content-Type", "text/html") return open(templateFile).read() % {"message": msg} @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) if self.require_auth: authV2(self.sydent, request) return self.do_validate_request(request) def do_validate_request(self, request: Request) -> JsonDict: """ Extracts information about a validation session from the request and attempts to validate that session. :param request: The request to extract information about the session from. :return: A dict with a "success" key which value indicates whether the validation succeeded. If the validation failed, this dict also includes a "errcode" and a "error" keys which include information about the failure. """ args = get_args(request, ("token", "sid", "client_secret")) sid = args["sid"] tokenString = args["token"] clientSecret = args["client_secret"] if not is_valid_client_secret(clientSecret): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } try: return self.sydent.validators.msisdn.validateSessionWithToken( sid, clientSecret, tokenString ) except IncorrectClientSecretException: request.setResponseCode(400) return { "success": False, "errcode": "M_INVALID_PARAM", "error": "Client secret does not match the one given when requesting the token", } except SessionExpiredException: request.setResponseCode(400) return { "success": False, "errcode": "M_SESSION_EXPIRED", "error": "This validation session has expired: call requestToken again", } except InvalidSessionIdException: request.setResponseCode(400) return { "success": False, "errcode": "M_INVALID_PARAM", "error": "The token doesn't match", } except IncorrectSessionTokenException: request.setResponseCode(404) return { "success": False, "errcode": "M_NO_VALID_SESSION", "error": "No session could be found with this sid", } def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/pubkeyservlets.py000066400000000000000000000041341414516477000226350ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from unpaddedbase64 import encode_base64 from sydent.db.invite_tokens import JoinTokenStore from sydent.http.servlets import get_args, jsonwrap from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class Ed25519Servlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) return {"public_key": pubKeyBase64} class PubkeyIsValidServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: args = get_args(request, ("public_key",)) pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) return {"valid": args["public_key"] == pubKeyBase64} class EphemeralPubkeyIsValidServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.joinTokenStore = JoinTokenStore(syd) @jsonwrap def render_GET(self, request: Request) -> JsonDict: args = get_args(request, ("public_key",)) publicKey = args["public_key"] return { "valid": self.joinTokenStore.validateEphemeralPublicKey(publicKey), } sydent-2.5.1/sydent/http/servlets/registerservlet.py000066400000000000000000000107111414516477000227750ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import urllib from http import HTTPStatus from json import JSONDecodeError from typing import TYPE_CHECKING, Dict from twisted.internet.error import ConnectError, DNSLookupError from twisted.web.client import ResponseFailed from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.httpclient import FederationHttpClient from sydent.http.servlets import asyncjsonwrap, get_args, send_cors from sydent.types import JsonDict from sydent.users.tokens import issueToken from sydent.util.stringutils import is_valid_matrix_server_name if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class RegisterServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd self.client = FederationHttpClient(self.sydent) @asyncjsonwrap async def render_POST(self, request: Request) -> JsonDict: """ Register with the Identity Server """ send_cors(request) args = get_args(request, ("matrix_server_name", "access_token")) matrix_server = args["matrix_server_name"].lower() if not is_valid_matrix_server_name(matrix_server): request.setResponseCode(400) return { "errcode": "M_INVALID_PARAM", "error": "matrix_server_name must be a valid Matrix server name (IP address or hostname)", } def federation_request_problem(error: str) -> Dict[str, str]: logger.warning(error) request.setResponseCode(HTTPStatus.INTERNAL_SERVER_ERROR) return { "errcode": "M_UNKNOWN", "error": error, } try: result = await self.client.get_json( "matrix://%s/_matrix/federation/v1/openid/userinfo?access_token=%s" % ( matrix_server, urllib.parse.quote(args["access_token"]), ), 1024 * 5, ) except (DNSLookupError, ConnectError, ResponseFailed) as e: return federation_request_problem( f"Unable to contact the Matrix homeserver ({type(e).__name__})" ) except JSONDecodeError: return federation_request_problem( "The Matrix homeserver returned invalid JSON" ) if "sub" not in result: return federation_request_problem( "The Matrix homeserver did not include 'sub' in its response", ) user_id = result["sub"] if not isinstance(user_id, str): return federation_request_problem( "The Matrix homeserver returned a malformed reply" ) user_id_components = user_id.split(":", 1) # Ensure there's a localpart and domain in the returned user ID. if len(user_id_components) != 2: return federation_request_problem( "The Matrix homeserver returned an invalid MXID" ) user_id_server = user_id_components[1] if not is_valid_matrix_server_name(user_id_server): return federation_request_problem( "The Matrix homeserver returned an invalid MXID" ) if user_id_server != matrix_server: return federation_request_problem( "The Matrix homeserver returned a MXID belonging to another homeserver" ) tok = issueToken(self.sydent, user_id) # XXX: `token` is correct for the spec, but we released with `access_token` # for a substantial amount of time. Serve both to make spec-compliant clients # happy. return { "access_token": tok, "token": tok, } def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/replication.py000066400000000000000000000157711414516477000220700ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import logging from typing import TYPE_CHECKING, List, cast import twisted.python.log from OpenSSL.crypto import X509 from twisted.internet.interfaces import ISSLTransport from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.hashing_metadata import HashingMetadataStore from sydent.db.peers import PeerStore from sydent.db.threepid_associations import GlobalAssociationStore, SignedAssociations from sydent.http.servlets import MatrixRestError, jsonwrap from sydent.threepid import threePidAssocFromDict from sydent.types import JsonDict from sydent.util import json_decoder from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.stringutils import normalise_address if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class ReplicationPushServlet(Resource): def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) @jsonwrap def render_POST(self, request: Request) -> JsonDict: # Cast safety: This request has an ISSLTransport because this servlet # is a resource under the ReplicationHttpsServer and nowhere else. request.transport = cast(ISSLTransport, request.transport) peerCert = cast(X509, request.transport.getPeerCertificate()) peerCertCn = peerCert.get_subject().commonName peerStore = PeerStore(self.sydent) peer = peerStore.getPeerByName(peerCertCn) if not peer: logger.warning( "Got connection from %s but no peer found by that name", peerCertCn ) raise MatrixRestError( 403, "M_UNKNOWN_PEER", "This peer is not known to this server" ) logger.info("Push connection made from peer %s", peer.servername) if ( not request.requestHeaders.hasHeader("Content-Type") # Type safety: the hasHeader call returned True, so getRawHeaders() # returns a nonempty list. or request.requestHeaders.getRawHeaders("Content-Type")[0] # type: ignore[index] != "application/json" ): logger.warning( "Peer %s made push connection with non-JSON content (type: %s)", peer.servername, # Type safety: the hasHeader call returned True, so getRawHeaders() # returns a nonempty list. request.requestHeaders.getRawHeaders("Content-Type")[0], # type: ignore[index] ) raise MatrixRestError(400, "M_NOT_JSON", "This endpoint expects JSON") try: # json.loads doesn't allow bytes in Python 3.5 inJson = json_decoder.decode(request.content.read().decode("UTF-8")) except ValueError: logger.warning( "Peer %s made push connection with malformed JSON", peer.servername ) raise MatrixRestError(400, "M_BAD_JSON", "Malformed JSON") if "sgAssocs" not in inJson: logger.warning( "Peer %s made push connection with no 'sgAssocs' key in JSON", peer.servername, ) raise MatrixRestError(400, "M_BAD_JSON", 'No "sgAssocs" key in JSON') failedIds: List[int] = [] globalAssocsStore = GlobalAssociationStore(self.sydent) # Ensure items are pulled out of the dictionary in order of origin_id. sg_assocs_raw: SignedAssociations = inJson.get("sgAssocs", {}) sg_assocs = sorted(sg_assocs_raw.items(), key=lambda k: int(k[0])) for originId, sgAssoc in sg_assocs: try: peer.verifySignedAssociation(sgAssoc) logger.debug( "Signed association from %s with origin ID %s verified", peer.servername, originId, ) # Don't bother adding if one has already failed: we add all of them or none so # we're only going to roll back the transaction anyway (but we continue to try # & verify the rest so we can give a complete list of the ones that don't # verify) if len(failedIds) > 0: continue assocObj = threePidAssocFromDict(sgAssoc) # ensure we are casefolding email addresses before hashing/storing assocObj.address = normalise_address(assocObj.address, assocObj.medium) if assocObj.mxid is not None: # Calculate the lookup hash with our own pepper for this association pepper = self.hashing_store.get_lookup_pepper() assert pepper is not None str_to_hash = " ".join( [assocObj.address, assocObj.medium, pepper], ) assocObj.lookup_hash = sha256_and_url_safe_base64(str_to_hash) # Add this association globalAssocsStore.addAssociation( assocObj, json.dumps(sgAssoc), peer.servername, originId, commit=False, ) else: logger.info( "Incoming deletion: removing associations for %s / %s", assocObj.medium, assocObj.address, ) globalAssocsStore.removeAssociation( assocObj.medium, assocObj.address ) logger.info( "Stored association origin ID %s from %s", originId, peer.servername ) except Exception: failedIds.append(originId) logger.warning( "Failed to verify signed association from %s with origin ID %s", peer.servername, originId, ) twisted.python.log.err() if len(failedIds) > 0: self.sydent.db.rollback() request.setResponseCode(400) return { "errcode": "M_VERIFICATION_FAILED", "error": "Verification failed for one or more associations", "failed_ids": failedIds, } else: self.sydent.db.commit() return {"success": True} sydent-2.5.1/sydent/http/servlets/store_invite_servlet.py000066400000000000000000000216241414516477000240270ustar00rootroot00000000000000# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import random import string from email.header import Header from http import HTTPStatus from typing import TYPE_CHECKING import nacl.signing from twisted.web.resource import Resource from twisted.web.server import Request from unpaddedbase64 import encode_base64 from sydent.db.invite_tokens import JoinTokenStore from sydent.db.threepid_associations import GlobalAssociationStore from sydent.http.auth import authV2 from sydent.http.servlets import MatrixRestError, get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util.emailutils import EmailAddressException, sendEmail from sydent.util.stringutils import MAX_EMAIL_ADDRESS_LENGTH, normalise_address if TYPE_CHECKING: from sydent.sydent import Sydent class StoreInviteServlet(Resource): def __init__(self, syd: "Sydent", require_auth: bool = False) -> None: self.sydent = syd self.random = random.SystemRandom() self.require_auth = require_auth @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) args = get_args( request, ( "medium", "address", "room_id", "sender", ), ) medium = args["medium"] address = args["address"] roomId = args["room_id"] sender = args["sender"] # ensure we are casefolding email address before storing normalised_address = normalise_address(address, medium) verified_sender = None if self.require_auth: account = authV2(self.sydent, request) verified_sender = sender if account.userId != sender: raise MatrixRestError(403, "M_UNAUTHORIZED", "'sender' doesn't match") globalAssocStore = GlobalAssociationStore(self.sydent) mxid = globalAssocStore.getMxid(medium, normalised_address) if mxid: request.setResponseCode(400) return { "errcode": "M_THREEPID_IN_USE", "error": "Binding already known", "mxid": mxid, } if medium != "email": request.setResponseCode(400) return { "errcode": "M_UNRECOGNIZED", "error": "Didn't understand medium '%s'" % (medium,), } if not (0 < len(address) <= MAX_EMAIL_ADDRESS_LENGTH): request.setResponseCode(400) return {"errcode": "M_INVALID_PARAM", "error": "Invalid email provided"} token = self._randomString(128) tokenStore = JoinTokenStore(self.sydent) ephemeralPrivateKey = nacl.signing.SigningKey.generate() ephemeralPublicKey = ephemeralPrivateKey.verify_key ephemeralPrivateKeyBase64 = encode_base64(ephemeralPrivateKey.encode(), True) ephemeralPublicKeyBase64 = encode_base64(ephemeralPublicKey.encode(), True) tokenStore.storeEphemeralPublicKey(ephemeralPublicKeyBase64) tokenStore.storeToken(medium, normalised_address, roomId, sender, token) # Variables to substitute in the template. substitutions = {} # Include all arguments sent via the request. for k, v in args.items(): if isinstance(v, str): substitutions[k] = v substitutions["token"] = token # Substitutions that the template requires, but are optional to provide # to the API. extra_substitutions = [ "sender_display_name", "token", "room_name", "bracketed_room_name", "room_avatar_url", "sender_avatar_url", "guest_user_id", "guest_access_token", ] for k in extra_substitutions: substitutions.setdefault(k, "") # For MSC3288 room type, prefer the stable field, but fallback to the # unstable field. if "room_type" not in substitutions: substitutions["room_type"] = substitutions.get( "org.matrix.msc3288.room_type", "" ) substitutions["bracketed_verified_sender"] = "" if verified_sender: substitutions["bracketed_verified_sender"] = "(%s) " % (verified_sender,) substitutions["ephemeral_private_key"] = ephemeralPrivateKeyBase64 if substitutions["room_name"] != "": substitutions["bracketed_room_name"] = "(%s) " % substitutions["room_name"] substitutions[ "web_client_location" ] = self.sydent.config.email.default_web_client_location if "org.matrix.web_client_location" in substitutions: substitutions["web_client_location"] = substitutions[ "org.matrix.web_client_location" ] if substitutions["room_type"] == "m.space": subject = self.sydent.config.email.invite_subject_space % substitutions else: subject = self.sydent.config.email.invite_subject % substitutions substitutions["subject_header_value"] = Header(subject, "utf8").encode() brand = self.sydent.brand_from_request(request) # self.sydent.config.email.invite_template is deprecated if self.sydent.config.email.invite_template is None: templateFile = self.sydent.get_branded_template( brand, "invite_template.eml", ) else: templateFile = self.sydent.config.email.invite_template try: sendEmail(self.sydent, templateFile, normalised_address, substitutions) except EmailAddressException: request.setResponseCode(HTTPStatus.BAD_REQUEST) return {"errcode": "M_INVALID_EMAIL", "error": "Invalid email address"} pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) baseUrl = "%s/_matrix/identity/api/v1" % ( self.sydent.config.http.server_http_url_base, ) keysToReturn = [] keysToReturn.append( { "public_key": pubKeyBase64, "key_validity_url": baseUrl + "/pubkey/isvalid", } ) keysToReturn.append( { "public_key": ephemeralPublicKeyBase64, "key_validity_url": baseUrl + "/pubkey/ephemeral/isvalid", } ) resp = { "token": token, "public_key": pubKeyBase64, "public_keys": keysToReturn, "display_name": self.redact_email_address(address), } return resp def redact_email_address(self, address: str) -> str: """ Redacts the content of a 3PID address. Redacts both the email's username and domain independently. :param address: The address to redact. :return: The redacted address. """ # Extract strings from the address username, domain = address.split("@", 1) # Obfuscate strings redacted_username = self._redact( username, self.sydent.config.email.username_obfuscate_characters ) redacted_domain = self._redact( domain, self.sydent.config.email.domain_obfuscate_characters ) return redacted_username + "@" + redacted_domain def _redact(self, s: str, characters_to_reveal: int) -> str: """ Redacts the content of a string, using a given amount of characters to reveal. If the string is shorter than the given threshold, redact it based on length. :param s: The string to redact. :param characters_to_reveal: How many characters of the string to leave before the '...' :return: The redacted string. """ # If the string is shorter than the defined threshold, redact based on length if len(s) <= characters_to_reveal: if len(s) > 5: return s[:3] + "..." if len(s) > 1: return s[0] + "..." return "..." # Otherwise truncate it and add an ellipses return s[:characters_to_reveal] + "..." def _randomString(self, length: int) -> str: """ Generate a random string of the given length. :param length: The length of the string to generate. :return: The generated string. """ return "".join(self.random.choice(string.ascii_letters) for _ in range(length)) sydent-2.5.1/sydent/http/servlets/termsservlet.py000066400000000000000000000051701414516477000223060ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.accounts import AccountStore from sydent.db.terms import TermsStore from sydent.http.auth import authV2 from sydent.http.servlets import MatrixRestError, get_args, jsonwrap, send_cors from sydent.terms.terms import get_terms from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class TermsServlet(Resource): isLeaf = True def __init__(self, syd: "Sydent") -> None: self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: """ Get the terms that must be agreed to in order to use this service Returns: Object describing the terms that require agreement """ send_cors(request) terms = get_terms(self.sydent) return terms.getForClient() @jsonwrap def render_POST(self, request: Request) -> JsonDict: """ Mark a set of terms and conditions as having been agreed to """ send_cors(request) account = authV2(self.sydent, request, False) args = get_args(request, ("user_accepts",)) user_accepts = args["user_accepts"] terms = get_terms(self.sydent) unknown_urls = list(set(user_accepts) - terms.getUrlSet()) if len(unknown_urls) > 0: raise MatrixRestError( 400, "M_UNKNOWN", "Unrecognised URLs: %s" % (", ".join(unknown_urls),) ) termsStore = TermsStore(self.sydent) termsStore.addAgreedUrls(account.userId, user_accepts) all_accepted_urls = termsStore.getAgreedUrls(account.userId) if terms.urlListIsSufficient(all_accepted_urls): accountStore = AccountStore(self.sydent) accountStore.setConsentVersion(account.userId, terms.getMasterVersion()) return {} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/threepidbindservlet.py000066400000000000000000000067361414516477000236260ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.valsession import ThreePidValSessionStore from sydent.http.auth import authV2 from sydent.http.servlets import MatrixRestError, get_args, jsonwrap, send_cors from sydent.types import JsonDict from sydent.util.stringutils import is_valid_client_secret from sydent.validators import ( IncorrectClientSecretException, InvalidSessionIdException, SessionExpiredException, SessionNotValidatedException, ) if TYPE_CHECKING: from sydent.sydent import Sydent class ThreePidBindServlet(Resource): def __init__(self, sydent: "Sydent", require_auth: bool = False) -> None: self.sydent = sydent self.require_auth = require_auth @jsonwrap def render_POST(self, request: Request) -> JsonDict: send_cors(request) account = None if self.require_auth: account = authV2(self.sydent, request) args = get_args(request, ("sid", "client_secret", "mxid")) sid = args["sid"] mxid = args["mxid"] clientSecret = args["client_secret"] if not is_valid_client_secret(clientSecret): raise MatrixRestError( 400, "M_INVALID_PARAM", "Invalid client_secret provided" ) if account: # This is a v2 API so only allow binding to the logged in user id if account.userId != mxid: raise MatrixRestError( 403, "M_UNAUTHORIZED", "This user is prohibited from binding to the mxid", ) try: valSessionStore = ThreePidValSessionStore(self.sydent) s = valSessionStore.getValidatedSession(sid, clientSecret) except (IncorrectClientSecretException, InvalidSessionIdException): # Return the same error for not found / bad client secret otherwise # people can get information about sessions without knowing the # secret. raise MatrixRestError( 404, "M_NO_VALID_SESSION", "No valid session was found matching that sid and client secret", ) except SessionExpiredException: raise MatrixRestError( 400, "M_SESSION_EXPIRED", "This validation session has expired: call requestToken again", ) except SessionNotValidatedException: raise MatrixRestError( 400, "M_SESSION_NOT_VALIDATED", "This validation session has not yet been completed", ) res = self.sydent.threepidBinder.addBinding(s.medium, s.address, mxid) return res def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/threepidunbindservlet.py000066400000000000000000000244551414516477000241670ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from http import HTTPStatus from typing import TYPE_CHECKING from signedjson.sign import SignatureVerifyException from twisted.internet import defer from twisted.internet.error import ConnectError, DNSLookupError from twisted.web import server from twisted.web.client import ResponseFailed from twisted.web.resource import Resource from twisted.web.server import Request from sydent.db.valsession import ThreePidValSessionStore from sydent.hs_federation.verifier import InvalidServerName, NoAuthenticationError from sydent.http.servlets import dict_to_json_bytes from sydent.types import JsonDict from sydent.util import json_decoder from sydent.util.stringutils import is_valid_client_secret from sydent.validators import ( IncorrectClientSecretException, InvalidSessionIdException, SessionNotValidatedException, ) if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class ThreePidUnbindServlet(Resource): def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def render_POST( self, request: Request ) -> object: # from the twisted docs: @type NOT_DONE_YET is an opaque object defer.ensureDeferred(self._async_render_POST(request)) return server.NOT_DONE_YET async def _async_render_POST(self, request: Request) -> None: try: try: # TODO: we should really validate that this gives us a dict, and # not some other json value like str, list, int etc # json.loads doesn't allow bytes in Python 3.5 body: JsonDict = json_decoder.decode( request.content.read().decode("UTF-8") ) except ValueError: request.setResponseCode(HTTPStatus.BAD_REQUEST) request.write( dict_to_json_bytes( {"errcode": "M_BAD_JSON", "error": "Malformed JSON"} ) ) request.finish() return missing = [k for k in ("threepid", "mxid") if k not in body] if len(missing) > 0: request.setResponseCode(HTTPStatus.BAD_REQUEST) msg = "Missing parameters: " + (",".join(missing)) request.write( dict_to_json_bytes({"errcode": "M_MISSING_PARAMS", "error": msg}) ) request.finish() return threepid = body["threepid"] mxid = body["mxid"] if "medium" not in threepid or "address" not in threepid: request.setResponseCode(HTTPStatus.BAD_REQUEST) request.write( dict_to_json_bytes( { "errcode": "M_MISSING_PARAMS", "error": "Threepid lacks medium / address", } ) ) request.finish() return # We now check for authentication in two different ways, depending # on the contents of the request. If the user has supplied "sid" # (the Session ID returned by Sydent during the original binding) # and "client_secret" fields, they are trying to prove that they # were the original author of the bind. We then check that what # they supply matches and if it does, allow the unbind. # # However if these fields are not supplied, we instead check # whether the request originated from a homeserver, and if so the # same homeserver that originally created the bind. We do this by # checking the signature of the request. If it all matches up, we # allow the unbind. # # Only one method of authentication is required. if "sid" in body and "client_secret" in body: sid = body["sid"] client_secret = body["client_secret"] if not is_valid_client_secret(client_secret): request.setResponseCode(HTTPStatus.BAD_REQUEST) request.write( dict_to_json_bytes( { "errcode": "M_INVALID_PARAM", "error": "Invalid client_secret provided", } ) ) request.finish() return valSessionStore = ThreePidValSessionStore(self.sydent) try: s = valSessionStore.getValidatedSession(sid, client_secret) except (IncorrectClientSecretException, InvalidSessionIdException): request.setResponseCode(HTTPStatus.UNAUTHORIZED) request.write( dict_to_json_bytes( { "errcode": "M_NO_VALID_SESSION", "error": "No valid session was found matching that sid and client secret", } ) ) request.finish() return except SessionNotValidatedException: request.setResponseCode(HTTPStatus.FORBIDDEN) request.write( dict_to_json_bytes( { "errcode": "M_SESSION_NOT_VALIDATED", "error": "This validation session has not yet been completed", } ) ) return if s.medium != threepid["medium"] or s.address != threepid["address"]: request.setResponseCode(HTTPStatus.FORBIDDEN) request.write( dict_to_json_bytes( { "errcode": "M_FORBIDDEN", "error": "Provided session information does not match medium/address combo", } ) ) request.finish() return else: try: origin_server_name = ( await self.sydent.sig_verifier.authenticate_request( request, body ) ) except SignatureVerifyException as ex: request.setResponseCode(HTTPStatus.UNAUTHORIZED) request.write( dict_to_json_bytes({"errcode": "M_FORBIDDEN", "error": str(ex)}) ) request.finish() return except NoAuthenticationError as ex: request.setResponseCode(HTTPStatus.UNAUTHORIZED) request.write( dict_to_json_bytes({"errcode": "M_FORBIDDEN", "error": str(ex)}) ) request.finish() return except InvalidServerName as ex: request.setResponseCode(HTTPStatus.BAD_REQUEST) request.write( dict_to_json_bytes( {"errcode": "M_INVALID_PARAM", "error": str(ex)} ) ) request.finish() return except (DNSLookupError, ConnectError, ResponseFailed) as e: msg = ( f"Unable to contact the Matrix homeserver to " f"authenticate request ({type(e).__name__})" ) logger.warning(msg) request.setResponseCode(HTTPStatus.INTERNAL_SERVER_ERROR) request.write( dict_to_json_bytes( { "errcode": "M_UNKNOWN", "error": msg, } ) ) request.finish() return except Exception: logger.exception("Exception whilst authenticating unbind request") request.setResponseCode(HTTPStatus.INTERNAL_SERVER_ERROR) request.write( dict_to_json_bytes( {"errcode": "M_UNKNOWN", "error": "Internal Server Error"} ) ) request.finish() return if not mxid.endswith(":" + origin_server_name): request.setResponseCode(HTTPStatus.FORBIDDEN) request.write( dict_to_json_bytes( { "errcode": "M_FORBIDDEN", "error": "Origin server name does not match mxid", } ) ) request.finish() return self.sydent.threepidBinder.removeBinding(threepid, mxid) request.write(dict_to_json_bytes({})) request.finish() except Exception as ex: logger.exception("Exception whilst handling unbind") request.setResponseCode(HTTPStatus.INTERNAL_SERVER_ERROR) request.write( dict_to_json_bytes({"errcode": "M_UNKNOWN", "error": str(ex)}) ) request.finish() sydent-2.5.1/sydent/http/servlets/v1_servlet.py000066400000000000000000000023341414516477000216400ustar00rootroot00000000000000# Copyright 2018 Travis Ralston # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.servlets import jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class V1Servlet(Resource): isLeaf = False def __init__(self, syd: "Sydent") -> None: Resource.__init__(self) self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: send_cors(request) return {} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/servlets/v2_servlet.py000066400000000000000000000023161414516477000216410ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING from twisted.web.resource import Resource from twisted.web.server import Request from sydent.http.servlets import jsonwrap, send_cors from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent class V2Servlet(Resource): isLeaf = False def __init__(self, syd: "Sydent") -> None: Resource.__init__(self) self.sydent = syd @jsonwrap def render_GET(self, request: Request) -> JsonDict: send_cors(request) return {} def render_OPTIONS(self, request: Request) -> bytes: send_cors(request) return b"" sydent-2.5.1/sydent/http/srvresolver.py000066400000000000000000000142061414516477000202740ustar00rootroot00000000000000# Copyright 2014-2016 OpenMarket Ltd # Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import random import time from typing import Awaitable, Callable, Dict, List, SupportsInt, Tuple import attr from twisted.internet.error import ConnectError from twisted.names import client, dns from twisted.names.dns import Record_SRV, RRHeader from twisted.names.error import DNSNameError, DomainError logger = logging.getLogger(__name__) SERVER_CACHE: Dict[bytes, List["Server"]] = {} @attr.s(frozen=True, slots=True, auto_attribs=True) class Server: """ Our record of an individual server which can be tried to reach a destination. Attributes: host (bytes): target hostname port (int): priority (int): weight (int): expires (int): when the cache should expire this record - in *seconds* since the epoch """ host: bytes port: int priority: int = 0 weight: int = 0 expires: int = 0 def pick_server_from_list(server_list: List[Server]) -> Tuple[bytes, int]: """Randomly choose a server from the server list. :param server_list: List of candidate servers. :returns a (host, port) pair for the chosen server. """ if not server_list: raise RuntimeError("pick_server_from_list called with empty list") # TODO: currently we only use the lowest-priority servers. We should maintain a # cache of servers known to be "down" and filter them out min_priority = min(s.priority for s in server_list) eligible_servers = list(s for s in server_list if s.priority == min_priority) total_weight = sum(s.weight for s in eligible_servers) target_weight = random.randint(0, total_weight) for s in eligible_servers: target_weight -= s.weight if target_weight <= 0: return s.host, s.port # this should be impossible. raise RuntimeError( "pick_server_from_list got to end of eligible server list.", ) # The signature of twisted.names.client.lookupService, if you omit the timeout # argument. This is unannotated, but we can deduce the signature as follows: # 1. Return type is the return type of # twisted.internet.interfaces.IResolver.lookupService. Its type annotation # is incorrect; its docstring says that tuple entries are a _list_ of RRHeaders, # but the annotation says the entries are individual RRHeaders. # 2. Because we're looking up SRV records, we know that the payload of the RRHeaders # will be Record_SRVs. I made RRHeader's stub generic over the type of its # payload to reflect this. But that's a lie compared to Twisted's actual # RRHeader Type, so we need to enclose these in strings. LookupService = Callable[ [str], Awaitable[ Tuple[ List["RRHeader[Record_SRV]"], List["RRHeader[object]"], List["RRHeader[object]"], ] ], ] class SrvResolver: """Interface to the dns client to do SRV lookups, with result caching. The default resolver in twisted.names doesn't do any caching (it has a CacheResolver, but the cache never gets populated), so we add our own caching layer here. :param dns_client: Twisted resolver impl :param cache: cache object :param get_time: Clock implementation. Should return seconds since the epoch. """ def __init__( self, lookup_service: LookupService = client.lookupService, cache: Dict[bytes, List[Server]] = SERVER_CACHE, get_time: Callable[[], SupportsInt] = time.time, ) -> None: self._lookup_service = lookup_service self._cache = cache self._get_time = get_time async def resolve_service(self, service_name: bytes) -> List["Server"]: """Look up a SRV record :param service_name: The record to look up. :returns a list of the SRV records, or an empty list if none found. """ now = int(self._get_time()) cache_entry = self._cache.get(service_name, None) if cache_entry: if all(s.expires > now for s in cache_entry): servers = list(cache_entry) return servers try: answers, _, _ = await self._lookup_service(service_name.decode()) except DNSNameError: # TODO: cache this. We can get the SOA out of the exception, and use # the negative-TTL value. return [] except DomainError as e: # We failed to resolve the name (other than a NameError) # Try something in the cache, else rereaise cache_entry = self._cache.get(service_name, None) if cache_entry: logger.warning( "Failed to resolve %r, falling back to cache. %r", service_name, e ) return list(cache_entry) else: raise e if ( len(answers) == 1 and answers[0].type == dns.SRV and answers[0].payload and answers[0].payload.target == dns.Name(b".") ): raise ConnectError("Service %s unavailable" % service_name.decode()) servers = [] for answer in answers: if answer.type != dns.SRV or not answer.payload: continue payload = answer.payload servers.append( Server( host=payload.target.name, port=payload.port, priority=payload.priority, weight=payload.weight, expires=now + answer.ttl, ) ) self._cache[service_name] = list(servers) return servers sydent-2.5.1/sydent/replication/000077500000000000000000000000001414516477000166555ustar00rootroot00000000000000sydent-2.5.1/sydent/replication/__init__.py000066400000000000000000000000001414516477000207540ustar00rootroot00000000000000sydent-2.5.1/sydent/replication/peer.py000066400000000000000000000263571414516477000201770ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import binascii import json import logging from abc import abstractmethod from typing import TYPE_CHECKING, Dict, Generic, Optional, Sequence, TypeVar import signedjson.key import signedjson.sign from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.python.failure import Failure from twisted.web.client import readBody from twisted.web.iweb import IResponse from unpaddedbase64 import decode_base64 from sydent.config.exceptions import ConfigError from sydent.db.hashing_metadata import HashingMetadataStore from sydent.db.threepid_associations import GlobalAssociationStore, SignedAssociations from sydent.threepid import threePidAssocFromDict from sydent.types import JsonDict from sydent.util import json_decoder from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.stringutils import normalise_address PushUpdateReturn = TypeVar("PushUpdateReturn") if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) SIGNING_KEY_ALGORITHM = "ed25519" class Peer(Generic[PushUpdateReturn]): def __init__(self, servername: str, pubkeys: Dict[str, str]): """ :param server_name: The peer's server name. :param pubkeys: The peer's public keys in a Dict[key_id, key_b64] """ self.servername = servername self.pubkeys = pubkeys self.is_being_pushed_to = False @abstractmethod def pushUpdates(self, sgAssocs: SignedAssociations) -> "Deferred[PushUpdateReturn]": """ :param sgAssocs: Map from originId to sgAssoc, where originId is the id on the creating server and sgAssoc is the json object of the signed association """ ... class LocalPeer(Peer[bool]): """ The local peer (ourselves: essentially copying from the local associations table to the global one) """ def __init__(self, sydent: "Sydent") -> None: super().__init__(sydent.config.general.server_name, {}) self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) globalAssocStore = GlobalAssociationStore(self.sydent) lastId = globalAssocStore.lastIdFromServer(self.servername) self.lastId = lastId if lastId is not None else -1 def pushUpdates(self, sgAssocs: SignedAssociations) -> "Deferred[bool]": """ Saves the given associations in the global associations store. Only stores an association if its ID is greater than the last seen ID. :param sgAssocs: The associations to save. :return: A deferred that succeeds with the value `True`. """ globalAssocStore = GlobalAssociationStore(self.sydent) for localId in sgAssocs: if localId > self.lastId: assocObj = threePidAssocFromDict(sgAssocs[localId]) # ensure we are casefolding email addresses assocObj.address = normalise_address(assocObj.address, assocObj.medium) if assocObj.mxid is not None: # Assign a lookup_hash to this association pepper = self.hashing_store.get_lookup_pepper() if not pepper: raise RuntimeError("No lookup_pepper in the database.") str_to_hash = " ".join( [ assocObj.address, assocObj.medium, pepper, ], ) assocObj.lookup_hash = sha256_and_url_safe_base64(str_to_hash) # We can probably skip verification for the local peer (although it could # be good as a sanity check) globalAssocStore.addAssociation( assocObj, json.dumps(sgAssocs[localId]), self.sydent.config.general.server_name, localId, ) else: globalAssocStore.removeAssociation( assocObj.medium, assocObj.address ) d = defer.succeed(True) return d class RemotePeer(Peer[IResponse]): def __init__( self, sydent: "Sydent", server_name: str, port: Optional[int], pubkeys: Dict[str, str], lastSentVersion: Optional[int], ) -> None: """ :param sydent: The current Sydent instance. :param server_name: The peer's server name. :param port: The peer's port. Only used if no replication url is configured. :param pubkeys: The peer's public keys in a dict[key_id, key_b64] :param lastSentVersion: The ID of the last association sent to the peer. """ super().__init__(server_name, pubkeys) self.sydent = sydent self.lastSentVersion = lastSentVersion # look up or build the replication URL replication_url = self.sydent.config.http.base_replication_urls.get(server_name) if replication_url is None: if not port: port = 1001 replication_url = "https://%s:%i" % (server_name, port) if replication_url[-1:] != "/": replication_url += "/" # Capture the interesting bit of the url for logging. self.replication_url_origin = replication_url replication_url += "_matrix/identity/replicate/v1/push" self.replication_url = replication_url # Get verify key for this peer # Check if their key is base64 or hex encoded pubkey = self.pubkeys[SIGNING_KEY_ALGORITHM] try: # Check for hex encoding int(pubkey, 16) # Decode hex into bytes pubkey_decoded = binascii.unhexlify(pubkey) logger.warning( "Peer public key of %s is hex encoded. Please update to base64 encoding", server_name, ) except ValueError: # Check for base64 encoding try: pubkey_decoded = decode_base64(pubkey) except Exception as e: raise ConfigError( "Unable to decode public key for peer %s: %s" % (server_name, e), ) self.verify_key = signedjson.key.decode_verify_key_bytes( SIGNING_KEY_ALGORITHM + ":", pubkey_decoded ) # Attach metadata self.verify_key.alg = SIGNING_KEY_ALGORITHM self.verify_key.version = 0 def verifySignedAssociation(self, assoc: JsonDict) -> None: """Verifies a signature on a signed association. Raises an exception if the signature is incorrect or couldn't be verified. :param assoc: A signed association. """ if "signatures" not in assoc: raise NoSignaturesException() key_ids = signedjson.sign.signature_ids(assoc, self.servername) if ( not key_ids or len(key_ids) == 0 or not key_ids[0].startswith(SIGNING_KEY_ALGORITHM + ":") ): e = NoMatchingSignatureException( foundSigs=assoc["signatures"].keys(), requiredServername=self.servername, ) raise e # Verify the JSON signedjson.sign.verify_signed_json(assoc, self.servername, self.verify_key) def pushUpdates(self, sgAssocs: SignedAssociations) -> "Deferred[IResponse]": """ Pushes the given associations to the peer. :param sgAssocs: The associations to push. :return: A deferred which results in the response to the push request. """ body = {"sgAssocs": sgAssocs} reqDeferred = self.sydent.replicationHttpsClient.postJson( self.replication_url, body ) if reqDeferred is None: raise RuntimeError(f"Unable to push sgAssocs to {self.replication_url}") # XXX: We'll also need to prune the deleted associations out of the # local associations table once they've been replicated to all peers # (ie. remove the record we kept in order to propagate the deletion to # other peers). updateDeferred: "Deferred[IResponse]" = defer.Deferred() reqDeferred.addCallback(self._pushSuccess, updateDeferred=updateDeferred) reqDeferred.addErrback(self._pushFailed, updateDeferred=updateDeferred) return updateDeferred def _pushSuccess( self, result: "IResponse", updateDeferred: "Deferred[IResponse]", ) -> None: """ Processes a successful push request. If the request resulted in a status code that's not a success, consider it a failure :param result: The HTTP response. :param updateDeferred: The deferred to make either succeed or fail depending on the status code. """ if result.code >= 200 and result.code < 300: updateDeferred.callback(result) else: d = readBody(result) d.addCallback(self._failedPushBodyRead, updateDeferred=updateDeferred) d.addErrback(self._pushFailed, updateDeferred=updateDeferred) def _failedPushBodyRead( self, body: bytes, updateDeferred: "Deferred[IResponse]" ) -> None: """ Processes a response body from a failed push request, then calls the error callback of the provided deferred. :param body: The response body. :param updateDeferred: The deferred to call the error callback of. """ errObj = json_decoder.decode(body.decode("utf8")) e = RemotePeerError(errObj) updateDeferred.errback(e) def _pushFailed( self, failure: Failure, updateDeferred: "Deferred[object]", ) -> None: """ Processes a failed push request, by calling the error callback of the given deferred with it. :param failure: The failure to process. :param updateDeferred: The deferred to call the error callback of. """ updateDeferred.errback(failure) return None class NoSignaturesException(Exception): pass class NoMatchingSignatureException(Exception): def __init__(self, foundSigs: Sequence[str], requiredServername: str): self.foundSigs = foundSigs self.requiredServername = requiredServername def __str__(self) -> str: return "Found signatures: %s, required server name: %s" % ( self.foundSigs, self.requiredServername, ) class RemotePeerError(Exception): def __init__(self, errorDict: JsonDict): self.errorDict = errorDict def __str__(self) -> str: return repr(self.errorDict) sydent-2.5.1/sydent/replication/pusher.py000066400000000000000000000111371414516477000205400ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, List, Tuple import twisted.internet.reactor import twisted.internet.task from twisted.internet import defer from sydent.db.peers import PeerStore from sydent.db.threepid_associations import LocalAssociationStore from sydent.replication.peer import LocalPeer, RemotePeer from sydent.util import time_msec if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) # Maximum amount of signed associations to replicate to a peer at a time ASSOCIATIONS_PUSH_LIMIT = 100 class Pusher: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.pushing = False self.peerStore = PeerStore(self.sydent) self.local_assoc_store = LocalAssociationStore(self.sydent) def setup(self) -> None: cb = twisted.internet.task.LoopingCall(Pusher.scheduledPush, self) cb.clock = self.sydent.reactor cb.start(10.0) def doLocalPush(self) -> None: """ Synchronously push local associations to this server (ie. copy them to globals table) The local server is essentially treated the same as any other peer except we don't do the network round-trip and this function can be used so the association goes into the global table before the http call returns (so clients know it will be available on at least the same ID server they used) """ localPeer = LocalPeer(self.sydent) signedAssocs, _ = self.local_assoc_store.getSignedAssociationsAfterId( localPeer.lastId, None ) localPeer.pushUpdates(signedAssocs) def scheduledPush(self) -> "defer.Deferred[List[Tuple[bool, None]]]": """Push pending updates to all known remote peers. To be called regularly. :returns a deferred.DeferredList of defers, one per peer we're pushing to that will resolve when pushing to that peer has completed, successfully or otherwise """ peers = self.peerStore.getAllPeers() # Push to all peers in parallel dl = [] for p in peers: dl.append(defer.ensureDeferred(self._push_to_peer(p))) return defer.DeferredList(dl) async def _push_to_peer(self, p: "RemotePeer") -> None: """ For a given peer, retrieves the list of associations that were created since the last successful push to this peer (limited to ASSOCIATIONS_PUSH_LIMIT) and sends them. :param p: The peer to send associations to. """ logger.debug("Looking for updates to push to %s", p.servername) # Check if a push operation is already active. If so, don't start another if p.is_being_pushed_to: logger.debug( "Waiting for %s to finish pushing...", p.replication_url_origin ) return p.is_being_pushed_to = True try: # Push associations ( assocs, latest_assoc_id, ) = self.local_assoc_store.getSignedAssociationsAfterId( p.lastSentVersion, ASSOCIATIONS_PUSH_LIMIT ) # If there are no updates left to send, break the loop if not assocs: return logger.info( "Pushing %d updates to %s", len(assocs), p.replication_url_origin ) result = await p.pushUpdates(assocs) self.peerStore.setLastSentVersionAndPokeSucceeded( p.servername, latest_assoc_id, time_msec() ) logger.info( "Pushed updates to %s with result %d %s", p.replication_url_origin, result.code, result.phrase, ) except Exception: logger.exception("Error pushing updates to %s", p.replication_url_origin) finally: # Whether pushing completed or an error occurred, signal that pushing has finished p.is_being_pushed_to = False sydent-2.5.1/sydent/sms/000077500000000000000000000000001414516477000151465ustar00rootroot00000000000000sydent-2.5.1/sydent/sms/__init__.py000066400000000000000000000000001414516477000172450ustar00rootroot00000000000000sydent-2.5.1/sydent/sms/openmarket.py000066400000000000000000000133101414516477000176630ustar00rootroot00000000000000# Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from base64 import b64encode from typing import TYPE_CHECKING, Dict, Optional, cast from twisted.web.http_headers import Headers from sydent.http.httpclient import SimpleHttpClient from sydent.sms.types import SendSMSBody, TypeOfNumber from sydent.types import JsonDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) API_BASE_URL = "https://smsc.openmarket.com/sms/v4/mt" # The Customer Integration Environment, where you can send # the same requests but it doesn't actually send any SMS. # Useful for testing. # API_BASE_URL = "http://smsc-cie.openmarket.com/sms/v4/mt" # The TON (ie. Type of Number) codes by type used in our config file TONS: Dict[str, TypeOfNumber] = { "long": 1, "short": 3, "alpha": 5, } def tonFromType(t: str) -> TypeOfNumber: """ Get the type of number from the originator's type. :param t: Type from the originator. :type t: str :return: The type of number. :rtype: int """ if t in TONS: return TONS[t] raise Exception("Unknown number type (%s) for originator" % t) class OpenMarketSMS: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.http_cli = SimpleHttpClient(sydent) async def sendTextSMS( self, body: str, dest: str, source: Optional[Dict[str, str]] = None ) -> None: """ Sends a text message with the given body to the given MSISDN. :param body: The message to send. :param dest: The destination MSISDN to send the text message to. """ send_body: SendSMSBody = { "mobileTerminate": { "message": {"content": body, "type": "text"}, "destination": { "address": dest, }, }, } if source: send_body["mobileTerminate"]["source"] = { "ton": tonFromType(source["type"]), "address": source["text"], } username = self.sydent.config.sms.api_username password = self.sydent.config.sms.api_password b64creds = b64encode(b"%s:%s" % (username, password)) req_headers = Headers( { b"Authorization": [b"Basic " + b64creds], b"Content-Type": [b"application/json"], } ) # Type safety: The case from a TypedDict to a regular Dict is required # because the two are deliberately not compatible. See # https://github.com/python/mypy/issues/4976 # for details, but in a nutshell: Dicts can have keys added or removed, # and that would break the invariants that a TypedDict is there to check. # The case below is safe because we never use send_body afterwards. resp, response_body = await self.http_cli.post_json_maybe_get_json( API_BASE_URL, cast(JsonDict, send_body), {"headers": req_headers} ) headers = dict(resp.headers.getAllRawHeaders()) request_id = None if b"X-Request-Id" in headers: request_id = headers[b"X-Request-Id"][0].decode("UTF-8") # Catch errors from the API. The documentation says a success code should be 202 # Accepted, but let's be broad here just in case and accept all 2xx response # codes. # # Relevant OpenMarket API documentation: # https://www.openmarket.com/docs/Content/apis/v4http/send-json.htm if resp.code < 200 or resp.code >= 300: if response_body is None or "error" not in response_body: raise Exception( "OpenMarket API responded with status %d (request ID: %s)" % ( resp.code, request_id, ), ) error = response_body["error"] raise Exception( "OpenMarket API responded with status %d (request ID: %s): %s" % ( resp.code, request_id, error, ), ) ticket_id = None if b"Location" not in headers: logger.error("Got response from sending SMS with no location header") else: # Nominally we should parse the URL, but we can just split on '/' since # we only care about the last part. value = headers[b"Location"][0].decode("UTF-8") parts = value.split("/") if len(parts) < 2: logger.error( "Got response from sending SMS with malformed location header: %s", value, ) else: ticket_id = parts[-1] logger.info( "Successfully sent SMS (ticket ID: %s, request ID %s), OpenMarket API" " responded with code %d", ticket_id, request_id, resp.code, ) logger.info( "Successfully sent SMS (ticket ID: %s, request ID %s), OpenMarket API" " responded with code %d", parts[-1], request_id, resp.code, ) sydent-2.5.1/sydent/sms/types.py000066400000000000000000000024241414516477000166660ustar00rootroot00000000000000# See "Request body" section of # https://www.openmarket.com/docs/Content/apis/v4http/send-json.htm from typing_extensions import Literal, TypedDict TypeOfNumber = Literal[1, 3, 5] class _MessageRequired(TypedDict): type: Literal["text", "hexEncodedText", "binary", "wapPush"] content: str class Message(_MessageRequired, total=False): charset: Literal["GSM", "Latin-1", "UTF-8", "UTF-16"] validityPeriod: int url: str mlc: Literal["reject", "truncate", "segment"] udh: bool class _DestinationRequired(TypedDict): address: str class Destination(_DestinationRequired, total=False): mobileOperatorId: int class _SourceRequired(TypedDict): address: str class Source(_SourceRequired, total=False): ton: TypeOfNumber class _MobileTerminateRequired(TypedDict): # OpenMarket says these are required fields destination: Destination message: Message class MobileTerminate(_MobileTerminateRequired, total=False): # And these are all optional. interaction: Literal["one-way", "two-way"] promotional: bool # Ignored, unless we're sending to India source: Source # The API also offers optional "options" and "delivery" keys, # which we don't use class SendSMSBody(TypedDict): mobileTerminate: MobileTerminate sydent-2.5.1/sydent/sydent.py000066400000000000000000000323061414516477000162300ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2018 New Vector Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import gc import logging import logging.handlers import os import sqlite3 from typing import Optional import attr import twisted.internet.reactor from signedjson.types import SigningKey from twisted.internet import address, task from twisted.internet.interfaces import ( IReactorCore, IReactorPluggableNameResolver, IReactorSSL, IReactorTCP, IReactorTime, ) from twisted.python import log from twisted.web.http import Request from zope.interface import Interface from sydent.config import SydentConfig from sydent.db.hashing_metadata import HashingMetadataStore from sydent.db.sqlitedb import SqliteDatabase from sydent.db.valsession import ThreePidValSessionStore from sydent.hs_federation.verifier import Verifier from sydent.http.httpcommon import SslComponents from sydent.http.httpsclient import ReplicationHttpsClient from sydent.http.httpserver import ( ClientApiHttpServer, InternalApiHttpServer, ReplicationHttpsServer, ) from sydent.http.servlets.accountservlet import AccountServlet from sydent.http.servlets.blindlysignstuffservlet import BlindlySignStuffServlet from sydent.http.servlets.bulklookupservlet import BulkLookupServlet from sydent.http.servlets.emailservlet import ( EmailRequestCodeServlet, EmailValidateCodeServlet, ) from sydent.http.servlets.getvalidated3pidservlet import GetValidated3pidServlet from sydent.http.servlets.hashdetailsservlet import HashDetailsServlet from sydent.http.servlets.logoutservlet import LogoutServlet from sydent.http.servlets.lookupservlet import LookupServlet from sydent.http.servlets.lookupv2servlet import LookupV2Servlet from sydent.http.servlets.msisdnservlet import ( MsisdnRequestCodeServlet, MsisdnValidateCodeServlet, ) from sydent.http.servlets.pubkeyservlets import ( Ed25519Servlet, EphemeralPubkeyIsValidServlet, PubkeyIsValidServlet, ) from sydent.http.servlets.registerservlet import RegisterServlet from sydent.http.servlets.replication import ReplicationPushServlet from sydent.http.servlets.store_invite_servlet import StoreInviteServlet from sydent.http.servlets.termsservlet import TermsServlet from sydent.http.servlets.threepidbindservlet import ThreePidBindServlet from sydent.http.servlets.threepidunbindservlet import ThreePidUnbindServlet from sydent.http.servlets.v1_servlet import V1Servlet from sydent.http.servlets.v2_servlet import V2Servlet from sydent.replication.pusher import Pusher from sydent.threepid.bind import ThreepidBinder from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.tokenutils import generateAlphanumericTokenOfLength from sydent.util.versionstring import get_version_string from sydent.validators.emailvalidator import EmailValidator from sydent.validators.msisdnvalidator import MsisdnValidator logger = logging.getLogger(__name__) class SydentReactor( IReactorCore, IReactorTCP, IReactorSSL, IReactorTime, IReactorPluggableNameResolver, Interface, ): pass class Sydent: def __init__( self, sydent_config: SydentConfig, reactor: SydentReactor = twisted.internet.reactor, # type: ignore[assignment] use_tls_for_federation: bool = True, ): self.config = sydent_config self.reactor = reactor self.use_tls_for_federation = use_tls_for_federation logger.info("Starting Sydent server") self.db: sqlite3.Connection = SqliteDatabase(self).db if self.config.general.sentry_enabled: import sentry_sdk sentry_sdk.init( dsn=self.config.general.sentry_dsn, release=get_version_string() ) with sentry_sdk.configure_scope() as scope: scope.set_tag("sydent_server_name", self.config.general.server_name) # workaround for https://github.com/getsentry/sentry-python/issues/803: we # disable automatic GC and run it periodically instead. gc.disable() cb = task.LoopingCall(run_gc) cb.clock = self.reactor cb.start(1.0) # See if a pepper already exists in the database # Note: This MUST be run before we start serving requests, otherwise lookups for # 3PID hashes may come in before we've completed generating them hashing_metadata_store = HashingMetadataStore(self) lookup_pepper = hashing_metadata_store.get_lookup_pepper() if not lookup_pepper: # No pepper defined in the database, generate one lookup_pepper = generateAlphanumericTokenOfLength(5) # Store it in the database and rehash 3PIDs hashing_metadata_store.store_lookup_pepper( sha256_and_url_safe_base64, lookup_pepper ) self.validators: Validators = Validators( EmailValidator(self), MsisdnValidator(self) ) self.keyring: Keyring = Keyring(self.config.crypto.signing_key) self.keyring.ed25519.alg = "ed25519" self.sig_verifier: Verifier = Verifier(self) self.servlets: Servlets = Servlets(self, lookup_pepper) self.threepidBinder: ThreepidBinder = ThreepidBinder(self) self.sslComponents: SslComponents = SslComponents(self) self.clientApiHttpServer = ClientApiHttpServer(self) self.replicationHttpsServer = ReplicationHttpsServer(self) self.replicationHttpsClient: ReplicationHttpsClient = ReplicationHttpsClient( self ) self.pusher: Pusher = Pusher(self) def run(self) -> None: self.clientApiHttpServer.setup() self.replicationHttpsServer.setup() self.pusher.setup() self.maybe_start_prometheus_server() # A dedicated validation session store just to clean up old sessions every N minutes self.cleanupValSession = ThreePidValSessionStore(self) cb = task.LoopingCall(self.cleanupValSession.deleteOldSessions) cb.clock = self.reactor cb.start(10 * 60.0) if self.config.http.internal_port is not None: internalport = self.config.http.internal_port interface = self.config.http.internal_bind_address self.internalApiHttpServer = InternalApiHttpServer(self) self.internalApiHttpServer.setup(interface, internalport) if self.config.general.pidfile: with open(self.config.general.pidfile, "w") as pidfile: pidfile.write(str(os.getpid()) + "\n") self.reactor.run() def maybe_start_prometheus_server(self) -> None: if self.config.general.prometheus_enabled: import prometheus_client prometheus_client.start_http_server( port=self.config.general.prometheus_port, addr=self.config.general.prometheus_addr, ) def ip_from_request(self, request: Request) -> Optional[str]: if self.config.http.obey_x_forwarded_for and request.requestHeaders.hasHeader( "X-Forwarded-For" ): # Type safety: hasHeaders returning True means that getRawHeaders # returns a nonempty list return request.requestHeaders.getRawHeaders("X-Forwarded-For")[0] # type: ignore[index] client = request.getClientAddress() if isinstance(client, (address.IPv4Address, address.IPv6Address)): return client.host else: return None def brand_from_request(self, request: Request) -> Optional[str]: """ If the brand GET parameter is passed, returns that as a string, otherwise returns None. :param request: The incoming request. :return: The brand to use or None if no hint is found. """ if b"brand" in request.args: return request.args[b"brand"][0].decode("utf-8") return None def get_branded_template( self, brand: Optional[str], template_name: str, ) -> str: """ Calculate a branded template filename to use. Attempt to use the hinted brand from the request if the brand is valid. Otherwise, fallback to the default brand. :param brand: The hint of which brand to use. :type brand: str or None :param template_name: The name of the template file to load. :type template_name: str :return: The template filename to use. :rtype: str """ # If a brand hint is provided, attempt to use it if it is valid. if brand: if brand not in self.config.general.valid_brands: brand = None # If the brand hint is not valid, or not provided, fallback to the default brand. if not brand: brand = self.config.general.default_brand root_template_path = self.config.general.templates_path # Grab jinja template if it exists if os.path.exists( os.path.join(root_template_path, brand, template_name + ".j2") ): return os.path.join(brand, template_name + ".j2") else: return os.path.join(root_template_path, brand, template_name) @attr.s(frozen=True, slots=True, auto_attribs=True) class Validators: email: EmailValidator msisdn: MsisdnValidator class Servlets: def __init__(self, sydent: Sydent, lookup_pepper: str): self.v1 = V1Servlet(sydent) self.v2 = V2Servlet(sydent) self.emailRequestCode = EmailRequestCodeServlet(sydent) self.emailRequestCodeV2 = EmailRequestCodeServlet(sydent, require_auth=True) self.emailValidate = EmailValidateCodeServlet(sydent) self.emailValidateV2 = EmailValidateCodeServlet(sydent, require_auth=True) self.msisdnRequestCode = MsisdnRequestCodeServlet(sydent) self.msisdnRequestCodeV2 = MsisdnRequestCodeServlet(sydent, require_auth=True) self.msisdnValidate = MsisdnValidateCodeServlet(sydent) self.msisdnValidateV2 = MsisdnValidateCodeServlet(sydent, require_auth=True) self.lookup = LookupServlet(sydent) self.bulk_lookup = BulkLookupServlet(sydent) self.hash_details = HashDetailsServlet(sydent, lookup_pepper) self.lookup_v2 = LookupV2Servlet(sydent, lookup_pepper) self.pubkey_ed25519 = Ed25519Servlet(sydent) self.pubkeyIsValid = PubkeyIsValidServlet(sydent) self.ephemeralPubkeyIsValid = EphemeralPubkeyIsValidServlet(sydent) self.threepidBind = ThreePidBindServlet(sydent) self.threepidBindV2 = ThreePidBindServlet(sydent, require_auth=True) self.threepidUnbind = ThreePidUnbindServlet(sydent) self.replicationPush = ReplicationPushServlet(sydent) self.getValidated3pid = GetValidated3pidServlet(sydent) self.getValidated3pidV2 = GetValidated3pidServlet(sydent, require_auth=True) self.storeInviteServlet = StoreInviteServlet(sydent) self.storeInviteServletV2 = StoreInviteServlet(sydent, require_auth=True) self.blindlySignStuffServlet = BlindlySignStuffServlet(sydent) self.blindlySignStuffServletV2 = BlindlySignStuffServlet( sydent, require_auth=True ) self.termsServlet = TermsServlet(sydent) self.accountServlet = AccountServlet(sydent) self.registerServlet = RegisterServlet(sydent) self.logoutServlet = LogoutServlet(sydent) @attr.s(frozen=True, slots=True, auto_attribs=True) class Keyring: ed25519: SigningKey def get_config_file_path() -> str: return os.environ.get("SYDENT_CONF", "sydent.conf") def run_gc() -> None: threshold = gc.get_threshold() counts = gc.get_count() for i in reversed(range(len(threshold))): if threshold[i] < counts[i]: gc.collect(i) def setup_logging(config: SydentConfig) -> None: """ Setup logging using the options specified in the config :param config: the configuration to use """ log_path = config.general.log_path log_level = config.general.log_level log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s" formatter = logging.Formatter(log_format) if log_path != "": handler: logging.StreamHandler = logging.handlers.TimedRotatingFileHandler( log_path, when="midnight", backupCount=365 ) handler.setFormatter(formatter) else: handler = logging.StreamHandler() handler.setFormatter(formatter) rootLogger = logging.getLogger("") rootLogger.setLevel(log_level) rootLogger.addHandler(handler) observer = log.PythonLoggingObserver() observer.start() if __name__ == "__main__": sydent_config = SydentConfig() sydent_config.parse_config_file(get_config_file_path()) setup_logging(sydent_config) syd = Sydent(sydent_config) syd.run() sydent-2.5.1/sydent/terms/000077500000000000000000000000001414516477000154765ustar00rootroot00000000000000sydent-2.5.1/sydent/terms/__init__.py000066400000000000000000000000001414516477000175750ustar00rootroot00000000000000sydent-2.5.1/sydent/terms/terms.py000066400000000000000000000121451414516477000172050ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Union import yaml from typing_extensions import TypedDict if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class TermConfig(TypedDict): master_version: str docs: Mapping[str, "Policy"] class Policy(TypedDict): version: str langs: Mapping[str, "LocalisedPolicy"] class LocalisedPolicy(TypedDict): name: str url: str VersionOrLang = Union[str, LocalisedPolicy] class Terms: def __init__(self, yamlObj: Optional[TermConfig]) -> None: """ :param yamlObj: The parsed YAML. """ self._rawTerms = yamlObj def getMasterVersion(self) -> Optional[str]: """ :return: The global (master) version of the terms, or None if there are no terms of service for this server. """ if self._rawTerms is None: return None return self._rawTerms["master_version"] def getForClient(self) -> Dict[str, Dict[str, Dict[str, VersionOrLang]]]: # Examples: # "policy" -> "terms_of_service", "version" -> "1.2.3" # "policy" -> "terms_of_service", "en" -> LocalisedPolicy """ :return: A dict which value for the "policies" key is a dict which contains the "docs" part of the terms' YAML. That nested dict is empty if no terms. """ policies: Dict[str, Dict[str, VersionOrLang]] = {} if self._rawTerms is not None: for docName, doc in self._rawTerms["docs"].items(): policies[docName] = { "version": doc["version"], } policies[docName].update(doc["langs"]) return {"policies": policies} def getUrlSet(self) -> Set[str]: """ :return: All the URLs for the terms in a set. Empty set if no terms. """ urls = set() if self._rawTerms is not None: for docName, doc in self._rawTerms["docs"].items(): for langName, lang in doc["langs"].items(): url = lang["url"] urls.add(url) return urls def urlListIsSufficient(self, urls: List[str]) -> bool: """ Checks whether the provided list of URLs (which represents the list of terms accepted by the user) is enough to allow the creation of the user's account. :param urls: The list of URLs of terms the user has accepted. :return: Whether the list is sufficient to allow the creation of the user's account. """ agreed = set() urlset = set(urls) if self._rawTerms is None: if urls: raise ValueError("No configured terms, but user accepted some terms") else: return True else: for docName, doc in self._rawTerms["docs"].items(): for lang in doc["langs"].values(): if lang["url"] in urlset: agreed.add(docName) break required = set(self._rawTerms["docs"].keys()) return agreed == required def get_terms(sydent: "Sydent") -> Terms: """Read and parse terms as specified in the config. Errors in reading, parsing and validating the config are raised as exceptions.""" # TODO - move some of this to parse_config termsPath = sydent.config.general.terms_path if termsPath == "": return Terms(None) with open(termsPath) as fp: termsYaml = yaml.safe_load(fp) # TODO use something like jsonschema instead of this handwritten code. if "master_version" not in termsYaml: raise Exception("No master version") elif not isinstance(termsYaml["master_version"], str): raise TypeError( f"master_version should be a string, not {termsYaml['master_version']!r}" ) if "docs" not in termsYaml: raise Exception("No 'docs' key in terms") for docName, doc in termsYaml["docs"].items(): if "version" not in doc: raise Exception("'%s' has no version" % (docName,)) if "langs" not in doc: raise Exception("'%s' has no langs" % (docName,)) for langKey, lang in doc["langs"].items(): if "name" not in lang: raise Exception("lang '%s' of doc %s has no name" % (langKey, docName)) if "url" not in lang: raise Exception("lang '%s' of doc %s has no url" % (langKey, docName)) return Terms(termsYaml) sydent-2.5.1/sydent/threepid/000077500000000000000000000000001414516477000161505ustar00rootroot00000000000000sydent-2.5.1/sydent/threepid/__init__.py000066400000000000000000000035011414516477000202600ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Any, Dict, Optional import attr def threePidAssocFromDict(d: Dict[str, Any]) -> "ThreepidAssociation": """Instantiates a ThreepidAssociation from the given dict.""" assoc = ThreepidAssociation( d["medium"], d["address"], None, # empty lookup_hash digest by default d["mxid"], d["ts"], d["not_before"], d["not_after"], ) return assoc @attr.s(slots=True, auto_attribs=True) class ThreepidAssociation: """ medium: The medium of the 3pid (eg. email) address: The identifier (eg. email address) lookup_hash: A hash digest of the 3pid. Can be a str or None mxid: The matrix ID the 3pid is associated with ts: The creation timestamp of this association, ms not_before: The timestamp, in ms, at which this association becomes valid not_after: The timestamp, in ms, at which this association ceases to be valid """ medium: str address: str lookup_hash: Optional[str] # Note: the next four fields were made optional in schema version 2. # See sydent.db.sqlitedb.SqliteDatabase._upgradeSchema mxid: Optional[str] ts: Optional[int] not_before: Optional[int] not_after: Optional[int] extra_fields: Dict[str, Any] = {} sydent-2.5.1/sydent/threepid/bind.py000066400000000000000000000176571414516477000174560ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import collections import logging import math from typing import TYPE_CHECKING, Any, Dict, Union import signedjson.sign from twisted.internet import defer from sydent.db.hashing_metadata import HashingMetadataStore from sydent.db.invite_tokens import JoinTokenStore from sydent.db.threepid_associations import LocalAssociationStore from sydent.http.httpclient import FederationHttpClient from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer from sydent.util import time_msec from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.stringutils import is_valid_matrix_server_name, normalise_address if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class ThreepidBinder: # the lifetime of a 3pid association THREEPID_ASSOCIATION_LIFETIME_MS = 100 * 365 * 24 * 60 * 60 * 1000 def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) def addBinding(self, medium: str, address: str, mxid: str) -> Dict[str, Any]: """ Binds the given 3pid to the given mxid. It's assumed that we have somehow validated that the given user owns the given 3pid :param medium: The medium of the 3PID to bind. :param address: The address of the 3PID to bind. :param mxid: The MXID to bind the 3PID to. :return: The signed association. """ # ensure we casefold email address before storing normalised_address = normalise_address(address, medium) localAssocStore = LocalAssociationStore(self.sydent) # Fill out the association details createdAt = time_msec() expires = createdAt + ThreepidBinder.THREEPID_ASSOCIATION_LIFETIME_MS # Hash the medium + address and store that hash for the purposes of # later lookups lookup_pepper = self.hashing_store.get_lookup_pepper() assert lookup_pepper is not None str_to_hash = " ".join( [normalised_address, medium, lookup_pepper], ) lookup_hash = sha256_and_url_safe_base64(str_to_hash) assoc = ThreepidAssociation( medium, normalised_address, lookup_hash, mxid, createdAt, createdAt, expires, ) localAssocStore.addOrUpdateAssociation(assoc) self.sydent.pusher.doLocalPush() joinTokenStore = JoinTokenStore(self.sydent) pendingJoinTokens = joinTokenStore.getTokens(medium, normalised_address) invites = [] # Widen the value type to Any: we're going to set the signed key # to point to a dict, but pendingJoinTokens yields Dict[str, str] token: Dict[str, Any] for token in pendingJoinTokens: token["mxid"] = mxid presigned = { "mxid": mxid, "token": token["token"], } token["signed"] = signedjson.sign.sign_json( presigned, self.sydent.config.general.server_name, self.sydent.keyring.ed25519, ) invites.append(token) if invites: assoc.extra_fields["invites"] = invites joinTokenStore.markTokensAsSent(medium, normalised_address) signer = Signer(self.sydent) sgassoc = signer.signedThreePidAssociation(assoc) defer.ensureDeferred(self._notify(sgassoc, 0)) return sgassoc def removeBinding(self, threepid: Dict[str, str], mxid: str) -> None: """ Removes the binding between a given 3PID and a given MXID. :param threepid: The 3PID of the binding to remove. :param mxid: The MXID of the binding to remove. """ # ensure we are casefolding email addresses threepid["address"] = normalise_address(threepid["address"], threepid["medium"]) localAssocStore = LocalAssociationStore(self.sydent) localAssocStore.removeAssociation(threepid, mxid) self.sydent.pusher.doLocalPush() async def _notify(self, assoc: Dict[str, Any], attempt: int) -> None: """ Sends data about a new association (and, if necessary, the associated invites) to the associated MXID's homeserver. :param assoc: The association to send down to the homeserver. :param attempt: The number of previous attempts to send this association. """ mxid = assoc["mxid"] mxid_parts = mxid.split(":", 1) if len(mxid_parts) != 2: logger.error( "Can't notify on bind for unparseable mxid %s. Not retrying.", assoc["mxid"], ) return matrix_server = mxid_parts[1] if not is_valid_matrix_server_name(matrix_server): logger.error( "MXID server part '%s' not a valid Matrix server name. Not retrying.", matrix_server, ) return post_url = "matrix://%s/_matrix/federation/v1/3pid/onbind" % (matrix_server,) logger.info("Making bind callback to: %s", post_url) # Make a POST to the chosen Synapse server http_client = FederationHttpClient(self.sydent) try: response = await http_client.post_json_get_nothing(post_url, assoc, {}) except Exception as e: self._notifyErrback(assoc, attempt, e) return # If the request failed, try again with exponential backoff if response.code != 200: self._notifyErrback( assoc, attempt, "Non-OK error code received (%d)" % response.code ) else: logger.info("Successfully notified on bind for %s" % (mxid,)) # Skip the deletion step if instructed so by the config. if not self.sydent.config.general.delete_tokens_on_bind: return # Only remove sent tokens when they've been successfully sent. try: joinTokenStore = JoinTokenStore(self.sydent) joinTokenStore.deleteTokens(assoc["medium"], assoc["address"]) logger.info( "Successfully deleted invite for %s from the store", assoc["address"], ) except Exception: logger.exception( "Couldn't remove invite for %s from the store", assoc["address"], ) def _notifyErrback( self, assoc: Dict[str, Any], attempt: int, error: Union[Exception, str] ) -> None: """ Handles errors when trying to send an association down to a homeserver by logging the error and scheduling a new attempt. :param assoc: The association to send down to the homeserver. :param attempt: The number of previous attempts to send this association. :param error: The error that was raised when trying to send the association. """ logger.warning( "Error notifying on bind for %s: %s - rescheduling", assoc["mxid"], error ) self.sydent.reactor.callLater( math.pow(2, attempt), self._notify, assoc, attempt + 1 ) # The below is lovingly ripped off of synapse/http/endpoint.py _Server = collections.namedtuple("_Server", "priority weight host port") sydent-2.5.1/sydent/threepid/signer.py000066400000000000000000000030401414516477000200060ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import TYPE_CHECKING, Any, Dict import signedjson.sign if TYPE_CHECKING: from sydent.sydent import Sydent from sydent.threepid import ThreepidAssociation class Signer: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def signedThreePidAssociation(self, assoc: "ThreepidAssociation") -> Dict[str, Any]: """ Signs a 3PID association. :param assoc: The association to sign. :return: A signed representation of the association. """ sgassoc = { "medium": assoc.medium, "address": assoc.address, "mxid": assoc.mxid, "ts": assoc.ts, "not_before": assoc.not_before, "not_after": assoc.not_after, } sgassoc.update(assoc.extra_fields) sgassoc = signedjson.sign.sign_json( sgassoc, self.sydent.config.general.server_name, self.sydent.keyring.ed25519 ) return sgassoc sydent-2.5.1/sydent/types.py000066400000000000000000000012151414516477000160610ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Any, Dict JsonDict = Dict[str, Any] sydent-2.5.1/sydent/users/000077500000000000000000000000001414516477000155055ustar00rootroot00000000000000sydent-2.5.1/sydent/users/__init__.py000066400000000000000000000000001414516477000176040ustar00rootroot00000000000000sydent-2.5.1/sydent/users/accounts.py000066400000000000000000000021701414516477000176760ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Optional class Account: def __init__( self, user_id: str, creation_ts: int, consent_version: Optional[str] ) -> None: """ :param user_id: The Matrix user ID for the account. :param creation_ts: The timestamp in milliseconds of the account's creation. :param consent_version: The version of the terms of services that the user last accepted. """ self.userId = user_id self.creationTs = creation_ts self.consentVersion = consent_version sydent-2.5.1/sydent/users/tokens.py000066400000000000000000000027051414516477000173660ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import time from typing import TYPE_CHECKING from sydent.db.accounts import AccountStore from sydent.util.tokenutils import generateAlphanumericTokenOfLength if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) def issueToken(sydent: "Sydent", user_id: str) -> str: """ Creates an account for the given Matrix user ID, then generates, saves and returns an access token for that account. :param sydent: The Sydent instance to use for storing the token. :param user_id: The Matrix user ID to issue a token for. :return: The access token for that account. """ accountStore = AccountStore(sydent) accountStore.storeAccount(user_id, int(time.time() * 1000), None) new_token = generateAlphanumericTokenOfLength(64) accountStore.addToken(user_id, new_token) return new_token sydent-2.5.1/sydent/util/000077500000000000000000000000001414516477000153215ustar00rootroot00000000000000sydent-2.5.1/sydent/util/__init__.py000066400000000000000000000021261414516477000174330ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import time from typing import NoReturn def time_msec() -> int: """ Get the current time in milliseconds. :return: The current time in milliseconds. """ return int(time.time() * 1000) def _reject_invalid_json(val: str) -> NoReturn: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) # a custom JSON decoder which will reject Python extensions to JSON. json_decoder = json.JSONDecoder(parse_constant=_reject_invalid_json) sydent-2.5.1/sydent/util/emailutils.py000066400000000000000000000123111414516477000200410ustar00rootroot00000000000000# Copyright 2014-2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import email.utils import logging import random import smtplib import string import urllib from html import escape from typing import TYPE_CHECKING, Dict import twisted.python.log from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) def sendEmail( sydent: "Sydent", templateFile: str, mailTo: str, substitutions: Dict[str, str], log_send_errors: bool = True, ) -> None: """ Sends an email with the given parameters. :param sydent: The Sydent instance to use when building the configuration to send the email with. :param templateFile: The filename of the template to use when building the body of the email. :param mailTo: The email address to send the email to. :param substitutions: The substitutions to use with the template. :param log_send_errors: Whether to log errors happening when sending an email. """ mailFrom = sydent.config.email.sender myHostname = sydent.config.email.host_name midRandom = "".join([random.choice(string.ascii_letters) for _ in range(16)]) messageid = "<%d%s@%s>" % (time_msec(), midRandom, myHostname) substitutions.update( { "messageid": messageid, "date": email.utils.formatdate(localtime=False), "to": mailTo, "from": mailFrom, } ) # use jinja for rendering if jinja templates are present if templateFile.endswith(".j2"): # We add randomize the multipart boundary to stop user input from # conflicting with it. substitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) template = sydent.config.general.template_environment.get_template(templateFile) mailString = template.render(substitutions) else: allSubstitutions = {} for k, v in substitutions.items(): allSubstitutions[k] = v allSubstitutions[k + "_forhtml"] = escape(v) allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) with open(templateFile) as template_file: mailString = template_file.read() % allSubstitutions try: check_valid_email_address(mailTo, allow_description=False) except EmailAddressException: logger.warning("Invalid email address %s", mailTo) raise mailServer = sydent.config.email.smtp_server mailPort = int(sydent.config.email.smtp_port) mailUsername = sydent.config.email.smtp_username mailPassword = sydent.config.email.smtp_password mailTLSMode = sydent.config.email.tls_mode logger.info( "Sending mail to %s with mail server: %s" % ( mailTo, mailServer, ) ) try: smtp: smtplib.SMTP if mailTLSMode == "SSL" or mailTLSMode == "TLS": smtp = smtplib.SMTP_SSL(mailServer, mailPort, myHostname) elif mailTLSMode == "STARTTLS": smtp = smtplib.SMTP(mailServer, mailPort, myHostname) smtp.starttls() else: smtp = smtplib.SMTP(mailServer, mailPort, myHostname) if mailUsername != "": smtp.login(mailUsername, mailPassword) # We're using the parsing above to do basic validation, but instead of # failing it may munge the address it returns. So we should *not* use # that parsed address, as it may not match any validation done # elsewhere. smtp.sendmail(mailFrom, mailTo, mailString.encode("utf-8")) smtp.quit() except Exception as origException: if log_send_errors: twisted.python.log.err() raise EmailSendException() from origException def check_valid_email_address(address: str, allow_description: bool) -> None: """Check the given string is a valid email address. Email addresses are complicated (see RFCs 5321, 5322 and 6531; plus https://www.netmeister.org/blog/email.html). This isn't a comprehensive validation; we defer to Python's stdlib. :raises EmailAddressException: if not. """ parsed_address = email.utils.parseaddr(address)[1] if parsed_address == "": raise EmailAddressException(f"Couldn't parse email address {address}.") if not allow_description and address != parsed_address: raise EmailAddressException( f"Parsing address ({address} yielded a different address" f"({parsed_address})" ) class EmailAddressException(Exception): pass class EmailSendException(Exception): pass sydent-2.5.1/sydent/util/hash.py000066400000000000000000000017661414516477000166300ustar00rootroot00000000000000# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import unpaddedbase64 def sha256_and_url_safe_base64(input_text: str) -> str: """SHA256 hash an input string, encode the digest as url-safe base64, and return :param input_text: string to hash :returns a sha256 hashed and url-safe base64 encoded digest """ digest = hashlib.sha256(input_text.encode()).digest() return unpaddedbase64.encode_base64(digest, urlsafe=True) sydent-2.5.1/sydent/util/ip_range.py000066400000000000000000000070211414516477000174570ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import itertools from typing import Iterable, Optional from netaddr import AddrFormatError, IPNetwork, IPSet # IP ranges that are considered private / unroutable / don't make sense. DEFAULT_IP_RANGE_BLACKLIST = [ # Localhost "127.0.0.0/8", # Private networks. "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", # Carrier grade NAT. "100.64.0.0/10", # Address registry. "192.0.0.0/24", # Link-local networks. "169.254.0.0/16", # Formerly used for 6to4 relay. "192.88.99.0/24", # Testing networks. "198.18.0.0/15", "192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24", # Multicast. "224.0.0.0/4", # Localhost "::1/128", # Link-local addresses. "fe80::/10", # Unique local addresses. "fc00::/7", # Testing networks. "2001:db8::/32", # Multicast. "ff00::/8", # Site-local addresses "fec0::/10", ] def generate_ip_set( ip_addresses: Optional[Iterable[str]], extra_addresses: Optional[Iterable[str]] = None, config_path: Optional[Iterable[str]] = None, ) -> IPSet: """ Generate an IPSet from a list of IP addresses or CIDRs. Additionally, for each IPv4 network in the list of IP addresses, also includes the corresponding IPv6 networks. This includes: * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1) * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2) * 6to4 Address (see RFC 3056, section 2) Args: ip_addresses: An iterable of IP addresses or CIDRs. extra_addresses: An iterable of IP addresses or CIDRs. config_path: The path in the configuration for error messages. Returns: A new IP set. """ result = IPSet() for ip in itertools.chain(ip_addresses or (), extra_addresses or ()): try: network = IPNetwork(ip) except AddrFormatError as e: raise Exception( "Invalid IP range provided: %s." % (ip,), config_path ) from e result.add(network) # It is possible that these already exist in the set, but that's OK. if ":" not in str(network): result.add(IPNetwork(network).ipv6(ipv4_compatible=True)) result.add(IPNetwork(network).ipv6(ipv4_compatible=False)) result.add(_6to4(network)) return result def _6to4(network: IPNetwork) -> IPNetwork: """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056.""" # 6to4 networks consist of: # * 2002 as the first 16 bits # * The first IPv4 address in the network hex-encoded as the next 32 bits # * The new prefix length needs to include the bits from the 2002 prefix. hex_network = hex(network.first)[2:] hex_network = ("0" * (8 - len(hex_network))) + hex_network return IPNetwork( "2002:%s:%s::/%d" % ( hex_network[:4], hex_network[4:], 16 + network.prefixlen, ) ) sydent-2.5.1/sydent/util/stringutils.py000066400000000000000000000100151414516477000202570ustar00rootroot00000000000000# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re from typing import Optional, Tuple from twisted.internet.abstract import isIPAddress, isIPv6Address # https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-register-email-requesttoken CLIENT_SECRET_REGEX = re.compile(r"^[0-9a-zA-Z\.=_\-]+$") # hostname/domain name # https://regex101.com/r/OyN1lg/2 hostname_regex = re.compile( r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$", flags=re.IGNORECASE, ) # it's unclear what the maximum length of an email address is. RFC3696 (as corrected # by errata) says: # the upper limit on address lengths should normally be considered to be 254. # # In practice, mail servers appear to be more tolerant and allow 400 characters # or so. Let's allow 500, which should be plenty for everyone. # MAX_EMAIL_ADDRESS_LENGTH = 500 def is_valid_client_secret(client_secret: str) -> bool: """Validate that a given string matches the client_secret regex defined by the spec :param client_secret: The client_secret to validate :return: Whether the client_secret is valid """ return ( 0 < len(client_secret) <= 255 and CLIENT_SECRET_REGEX.match(client_secret) is not None ) def is_valid_hostname(string: str) -> bool: """Validate that a given string is a valid hostname or domain name. For domain names, this only validates that the form is right (for instance, it doesn't check that the TLD is valid). :param string: The string to validate :return: Whether the input is a valid hostname """ return hostname_regex.match(string) is not None def parse_server_name(server_name: str) -> Tuple[str, Optional[str]]: """Split a server name into host/port parts. No validation is done on the host part. The port part is validated to be a valid port number. Args: server_name: server name to parse Returns: host/port parts. Raises: ValueError if the server name could not be parsed. """ try: if server_name[-1] == "]": # ipv6 literal, hopefully return server_name, None host_port = server_name.rsplit(":", 1) host = host_port[0] port = host_port[1] if host_port[1:] else None if port: port_num = int(port) # exclude things like '08090' or ' 8090' if port != str(port_num) or not (1 <= port_num < 65536): raise ValueError("Invalid port") return host, port except Exception: raise ValueError("Invalid server name '%s'" % server_name) def is_valid_matrix_server_name(string: str) -> bool: """Validate that the given string is a valid Matrix server name. A string is a valid Matrix server name if it is one of the following, plus an optional port: a. IPv4 address b. IPv6 literal (`[IPV6_ADDRESS]`) c. A valid hostname :param string: The string to validate :return: Whether the input is a valid Matrix server name """ try: host, port = parse_server_name(string) except ValueError: return False valid_ipv4_addr = isIPAddress(host) valid_ipv6_literal = ( host[0] == "[" and host[-1] == "]" and isIPv6Address(host[1:-1]) ) return valid_ipv4_addr or valid_ipv6_literal or is_valid_hostname(host) def normalise_address(address: str, medium: str) -> str: if medium == "email": return address.casefold() else: return address sydent-2.5.1/sydent/util/tokenutils.py000066400000000000000000000034261414516477000201010ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import random import string r = random.SystemRandom() def generateTokenForMedium(medium: str) -> str: """ Generates a token of a different format depending on the medium, a 32 characters alphanumeric one if the medium is email, a 6 characters numeric one otherwise. :param medium: The medium to generate a token for. :return: The generated token. """ if medium == "email": return generateAlphanumericTokenOfLength(32) else: return generateNumericTokenOfLength(6) def generateNumericTokenOfLength(length: int) -> str: """ Generates a token of the given length with the character set [0-9]. :param length: The length of the token to generate. :return: The generated token. """ return "".join([r.choice(string.digits) for _ in range(length)]) def generateAlphanumericTokenOfLength(length: int) -> str: """ Generates a token of the given length with the character set [a-zA-Z0-9]. :param length: The length of the token to generate. :return: The generated token. """ return "".join( [ r.choice(string.digits + string.ascii_lowercase + string.ascii_uppercase) for _ in range(length) ] ) sydent-2.5.1/sydent/util/ttlcache.py000066400000000000000000000112351414516477000174640ustar00rootroot00000000000000# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import enum import logging import time from typing import Callable, Dict, Generic, Tuple, TypeVar, Union import attr from sortedcontainers import SortedList from typing_extensions import Literal logger = logging.getLogger(__name__) class Sentinel(enum.Enum): token = enum.auto() K = TypeVar("K") V = TypeVar("V") class TTLCache(Generic[K, V]): """A key/value cache implementation where each entry has its own TTL""" def __init__(self, cache_name: str, timer: Callable[[], float] = time.time): self._data: Dict[K, _CacheEntry[K, V]] = {} # the _CacheEntries, sorted by expiry time self._expiry_list: SortedList[_CacheEntry] = SortedList() self._timer = timer def set(self, key: K, value: V, ttl: float) -> None: """Add/update an entry in the cache :param key: Key for this entry. :param value: Value for this entry. :param ttl: TTL for this entry, in seconds. """ expiry = self._timer() + ttl self.expire() e = self._data.pop(key, Sentinel.token) if e != Sentinel.token: self._expiry_list.remove(e) entry = _CacheEntry(expiry_time=expiry, key=key, value=value) self._data[key] = entry self._expiry_list.add(entry) def get( self, key: K, default: Union[V, Literal[Sentinel.token]] = Sentinel.token ) -> V: """Get a value from the cache :param key: The key to look up. :param default: default value to return, if key is not found. If not set, and the key is not found, a KeyError will be raised. :returns a value from the cache, or the default. """ self.expire() e = self._data.get(key, Sentinel.token) if e is Sentinel.token: if default is Sentinel.token: raise KeyError(key) return default return e.value def get_with_expiry(self, key: K) -> Tuple[V, float]: """Get a value, and its expiry time, from the cache :param key: key to look up :returns The value from the cache, and the expiry time. :rtype: Tuple[Any, float] Raises: KeyError if the entry is not found """ self.expire() try: e = self._data[key] except KeyError: raise return e.value, e.expiry_time def pop( self, key: K, default: Union[V, Literal[Sentinel.token]] = Sentinel.token ) -> V: """Remove a value from the cache If key is in the cache, remove it and return its value, else return default. If default is not given and key is not in the cache, a KeyError is raised. :param key: key to look up :param default: default value to return, if key is not found. If not set, and the key is not found, a KeyError will be raised :returns a value from the cache, or the default """ self.expire() e = self._data.pop(key, Sentinel.token) if e is Sentinel.token: if default == Sentinel.token: raise KeyError(key) return default self._expiry_list.remove(e) return e.value def __getitem__(self, key: K) -> V: return self.get(key) def __delitem__(self, key: K) -> None: self.pop(key) def __contains__(self, key: K) -> bool: return key in self._data def __len__(self) -> int: self.expire() return len(self._data) def expire(self) -> None: """Run the expiry on the cache. Any entries whose expiry times are due will be removed """ now = self._timer() while self._expiry_list: first_entry = self._expiry_list[0] if first_entry.expiry_time - now > 0.0: break del self._data[first_entry.key] del self._expiry_list[0] @attr.s(frozen=True) class _CacheEntry(Generic[K, V]): """TTLCache entry""" # expiry_time is the first attribute, so that entries are sorted by expiry. expiry_time: float = attr.ib() key: K = attr.ib() value: V = attr.ib() sydent-2.5.1/sydent/util/versionstring.py000066400000000000000000000060521414516477000206120ustar00rootroot00000000000000import logging import os import subprocess import sydent logger = logging.getLogger(__name__) def get_version_string() -> str: """Calculate a git-aware version string for sydent. The version string is `sydent@x.y.z`, where `x.y.z` comes from `sydent.__version__`. If the `sydent` module belongs to a git repository on disk, we the current branch, tag and commit hash, plus a flag to indicate if the working tree is dirty. An example probably illustrates best. Testing locally, I get `sydent@2.4.6 (b=dmr/version-in-sentry,05e1fa8,dirty)`. If this wasn't done from within the repository, I'd just get `sydent@2.4.6`. Implementation nicked from Synapse. """ version_string = sydent.__version__ try: cwd = os.path.dirname(os.path.abspath(sydent.__file__)) try: git_branch = ( subprocess.check_output( ["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL, cwd=cwd, ) .strip() .decode("ascii") ) git_branch = "b=" + git_branch except (subprocess.CalledProcessError, FileNotFoundError): # FileNotFoundError can arise when git is not installed git_branch = "" try: git_tag = ( subprocess.check_output( ["git", "describe", "--exact-match"], stderr=subprocess.DEVNULL, cwd=cwd, ) .strip() .decode("ascii") ) git_tag = "t=" + git_tag except (subprocess.CalledProcessError, FileNotFoundError): git_tag = "" try: git_commit = ( subprocess.check_output( ["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL, cwd=cwd, ) .strip() .decode("ascii") ) except (subprocess.CalledProcessError, FileNotFoundError): git_commit = "" try: dirty_string = "-this_is_a_dirty_checkout" is_dirty = ( subprocess.check_output( ["git", "describe", "--dirty=" + dirty_string], stderr=subprocess.DEVNULL, cwd=cwd, ) .strip() .decode("ascii") .endswith(dirty_string) ) git_dirty = "dirty" if is_dirty else "" except (subprocess.CalledProcessError, FileNotFoundError): git_dirty = "" if git_branch or git_tag or git_commit or git_dirty: git_version = ",".join( s for s in (git_branch, git_tag, git_commit, git_dirty) if s ) version_string = f"sydent@{sydent.__version__} ({git_version})" except Exception as e: logger.info("Failed to check for git repository: %s", e) return version_string sydent-2.5.1/sydent/validators/000077500000000000000000000000001414516477000165145ustar00rootroot00000000000000sydent-2.5.1/sydent/validators/__init__.py000066400000000000000000000027161414516477000206330ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import attr # how long a user can wait before validating a session after starting it THREEPID_SESSION_VALIDATION_TIMEOUT_MS = 24 * 60 * 60 * 1000 # how long we keep sessions for after they've been validated THREEPID_SESSION_VALID_LIFETIME_MS = 24 * 60 * 60 * 1000 @attr.s(frozen=True, slots=True, auto_attribs=True) class ValidationSession: id: int medium: str address: str client_secret: str validated: bool mtime: int @attr.s(frozen=True, slots=True, auto_attribs=True) class TokenInfo: token: str send_attempt_number: int class IncorrectClientSecretException(Exception): pass class SessionExpiredException(Exception): pass class InvalidSessionIdException(Exception): pass class IncorrectSessionTokenException(Exception): pass class SessionNotValidatedException(Exception): pass class DestinationRejectedException(Exception): pass sydent-2.5.1/sydent/validators/common.py000066400000000000000000000044661414516477000203700ustar00rootroot00000000000000import logging from typing import TYPE_CHECKING, Dict from sydent.db.valsession import ThreePidValSessionStore from sydent.util import time_msec from sydent.validators import ( THREEPID_SESSION_VALIDATION_TIMEOUT_MS, IncorrectClientSecretException, IncorrectSessionTokenException, InvalidSessionIdException, SessionExpiredException, ) if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) def validateSessionWithToken( sydent: "Sydent", sid: int, clientSecret: str, token: str ) -> Dict[str, bool]: """ Attempt to validate a session, identified by the sid, using the token from out-of-band. The client secret is given to prevent attempts to guess the token for a sid. :param sid: The ID of the session to validate. :param clientSecret: The client secret to validate. :param token: The token to validate. :return: A dict with a "success" key which is True if the session was successfully validated, False otherwise. :raise IncorrectClientSecretException: The provided client_secret is incorrect. :raise SessionExpiredException: The session has expired. :raise InvalidSessionIdException: The session ID couldn't be matched with an existing session. :raise IncorrectSessionTokenException: The provided token is incorrect """ valSessionStore = ThreePidValSessionStore(sydent) result = valSessionStore.getTokenSessionById(sid) if not result: logger.info("Session ID %s not found", sid) raise InvalidSessionIdException() session, token_info = result if not clientSecret == session.client_secret: logger.info("Incorrect client secret", sid) raise IncorrectClientSecretException() if session.mtime + THREEPID_SESSION_VALIDATION_TIMEOUT_MS < time_msec(): logger.info("Session expired") raise SessionExpiredException() # TODO once we can validate the token oob # if tokenObj.validated and clientSecret == tokenObj.clientSecret: # return True if token_info.token == token: logger.info("Setting session %s as validated", session.id) valSessionStore.setValidated(session.id, True) return {"success": True} else: logger.info("Incorrect token submitted") raise IncorrectSessionTokenException() sydent-2.5.1/sydent/validators/emailvalidator.py000066400000000000000000000125341414516477000220700ustar00rootroot00000000000000# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import urllib from typing import TYPE_CHECKING, Dict, Optional from sydent.db.valsession import ThreePidValSessionStore from sydent.util import time_msec from sydent.util.emailutils import sendEmail from sydent.validators import common if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class EmailValidator: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def requestToken( self, emailAddress: str, clientSecret: str, sendAttempt: int, nextLink: Optional[str], ipaddress: Optional[str] = None, brand: Optional[str] = None, ) -> int: """ Creates or retrieves a validation session and sends an email to the corresponding email address with a token to use to verify the association. :param emailAddress: The email address to send the email to. :param clientSecret: The client secret to use. :param sendAttempt: The current send attempt. :param nextLink: The link to redirect the user to once they have completed the validation. :param ipaddress: The requester's IP address. :param brand: A hint at a brand from the request. :return: The ID of the session created (or of the existing one if any) """ valSessionStore = ThreePidValSessionStore(self.sydent) valSession, token_info = valSessionStore.getOrCreateTokenSession( medium="email", address=emailAddress, clientSecret=clientSecret ) valSessionStore.setMtime(valSession.id, time_msec()) # self.sydent.config.email.template is deprecated if self.sydent.config.email.template is None: templateFile = self.sydent.get_branded_template( brand, "verification_template.eml", ) else: templateFile = self.sydent.config.email.template if token_info.send_attempt_number >= sendAttempt: logger.info( "Not mailing code because current send attempt (%d) is not less than given send attempt (%s)", sendAttempt, token_info.send_attempt_number, ) return valSession.id ipstring = ipaddress if ipaddress else "an unknown location" substitutions = { "ipaddress": ipstring, "link": self.makeValidateLink( valSession.id, token_info.token, clientSecret, nextLink ), "token": token_info.token, } logger.info( "Attempting to mail code %s (nextLink: %s) to %s", token_info.token, nextLink, emailAddress, ) sendEmail(self.sydent, templateFile, emailAddress, substitutions) valSessionStore.setSendAttemptNumber(valSession.id, sendAttempt) return valSession.id def makeValidateLink( self, session_id: int, token: str, clientSecret: str, nextLink: Optional[str], ) -> str: """ Creates a validation link that can be sent via email to the user. :param session_id: The current validation session's ID. :param token: The token to make a link for. :param clientSecret: The client secret to include in the link. :param nextLink: The link to redirect the user to once they have completed the validation. :return: The validation link. """ base = self.sydent.config.http.server_http_url_base link = "%s/_matrix/identity/api/v1/validate/email/submitToken?token=%s&client_secret=%s&sid=%d" % ( base, urllib.parse.quote(token), urllib.parse.quote(clientSecret), session_id, ) if nextLink: # manipulate the nextLink to add the sid, because # the caller won't get it until we send a response, # by which time we've sent the mail. if "?" in nextLink: nextLink += "&" else: nextLink += "?" nextLink += "sid=" + urllib.parse.quote(str(session_id)) link += "&nextLink=%s" % (urllib.parse.quote(nextLink)) return link def validateSessionWithToken( self, sid: int, clientSecret: str, token: str ) -> Dict[str, bool]: """ Validates the session with the given ID. :param sid: The ID of the session to validate. :param clientSecret: The client secret to validate. :param token: The token to validate. :return: A dict with a "success" key which is True if the session was successfully validated, False otherwise. """ return common.validateSessionWithToken(self.sydent, sid, clientSecret, token) sydent-2.5.1/sydent/validators/msisdnvalidator.py000066400000000000000000000124241414516477000222740ustar00rootroot00000000000000# Copyright 2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from typing import TYPE_CHECKING, Dict, Optional import phonenumbers from sydent.db.valsession import ThreePidValSessionStore from sydent.sms.openmarket import OpenMarketSMS from sydent.util import time_msec from sydent.validators import DestinationRejectedException, common if TYPE_CHECKING: from sydent.sydent import Sydent logger = logging.getLogger(__name__) class MsisdnValidator: def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.omSms = OpenMarketSMS(sydent) # cache originators & sms rules from config file self.originators = self.sydent.config.sms.originators self.smsRules = self.sydent.config.sms.smsRules async def requestToken( self, phoneNumber: phonenumbers.PhoneNumber, clientSecret: str, send_attempt: int, brand: Optional[str] = None, ) -> int: """ Creates or retrieves a validation session and sends an text message to the corresponding phone number address with a token to use to verify the association. :param phoneNumber: The phone number to send the email to. :param clientSecret: The client secret to use. :param send_attempt: The current send attempt. :param brand: A hint at a brand from the request. :return: The ID of the session created (or of the existing one if any) """ if str(phoneNumber.country_code) in self.smsRules: action = self.smsRules[str(phoneNumber.country_code)] if action == "reject": raise DestinationRejectedException() valSessionStore = ThreePidValSessionStore(self.sydent) msisdn = phonenumbers.format_number( phoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] valSession, token_info = valSessionStore.getOrCreateTokenSession( medium="msisdn", address=msisdn, clientSecret=clientSecret ) valSessionStore.setMtime(valSession.id, time_msec()) if token_info.send_attempt_number >= send_attempt: logger.info( "Not texting code because current send attempt (%d) is not less than given send attempt (%s)", send_attempt, token_info.send_attempt_number, ) return valSession.id smsBodyTemplate = self.sydent.config.sms.body_template originator = self.getOriginator(phoneNumber) logger.info( "Attempting to text code %s to %s (country %d) with originator %s", token_info.token, msisdn, phoneNumber.country_code, originator, ) smsBody = smsBodyTemplate.format(token=token_info.token) await self.omSms.sendTextSMS(smsBody, msisdn, originator) valSessionStore.setSendAttemptNumber(valSession.id, send_attempt) return valSession.id def getOriginator( self, destPhoneNumber: phonenumbers.PhoneNumber ) -> Dict[str, str]: """ Gets an originator for a given phone number. :param destPhoneNumber: The phone number to find the originator for. :return: The originator (a dict with a "type" key and a "text" key). """ countryCode = str(destPhoneNumber.country_code) origs = [ { "type": "alpha", "text": "Matrix", } ] if countryCode in self.originators: origs = self.originators[countryCode] elif "default" in self.originators: origs = self.originators["default"] # deterministically pick an originator from the list of possible # originators, so if someone requests multiple codes, they come from # a consistent number (if there's any chance that some originators are # more likley to work than others, we may want to change, but it feels # like this should be something other than just picking one randomly). msisdn = phonenumbers.format_number( destPhoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] return origs[sum(int(i) for i in msisdn) % len(origs)] def validateSessionWithToken( self, sid: int, clientSecret: str, token: str ) -> Dict[str, bool]: """ Validates the session with the given ID. :param sid: The ID of the session to validate. :param clientSecret: The client secret to validate. :param token: The token to validate. :return: A dict with a "success" key which is True if the session was successfully validated, False otherwise. """ return common.validateSessionWithToken(self.sydent, sid, clientSecret, token) sydent-2.5.1/terms.sample.yaml000066400000000000000000000011041414516477000163300ustar00rootroot00000000000000master_version: "master_1_1" docs: terms_of_service: version: "2.0" langs: en: name: "Terms of Service" url: "https://example.org/somewhere/terms-2.0-en.html" fr: name: "Conditions d'utilisation" url: "https://example.org/somewhere/terms-2.0-fr.html" privacy_policy: version: "1.2" langs: en: name: "Privacy Policy" url: "https://example.org/somewhere/privacy-1.2-en.html" fr: name: "Politique de confidentialité" url: "https://example.org/somewhere/privacy-1.2-fr.html" sydent-2.5.1/tests/000077500000000000000000000000001414516477000142005ustar00rootroot00000000000000sydent-2.5.1/tests/__init__.py000066400000000000000000000000001414516477000162770ustar00rootroot00000000000000sydent-2.5.1/tests/test_auth.py000066400000000000000000000044501414516477000165550ustar00rootroot00000000000000# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from twisted.trial import unittest from sydent.http.auth import tokenFromRequest from tests.utils import make_request, make_sydent class AuthTestCase(unittest.TestCase): """Tests Sydent's auth code""" def setUp(self): # Create a new sydent self.sydent = make_sydent() self.test_token = "testingtoken" # Inject a fake OpenID token into the database cur = self.sydent.db.cursor() cur.execute( "INSERT INTO accounts (user_id, created_ts, consent_version)" "VALUES (?, ?, ?)", ("@bob:localhost", 101010101, "asd"), ) cur.execute( "INSERT INTO tokens (user_id, token)" "VALUES (?, ?)", ("@bob:localhost", self.test_token), ) self.sydent.db.commit() def test_can_read_token_from_headers(self): """Tests that Sydent correctly extracts an auth token from request headers""" self.sydent.run() request, _ = make_request( self.sydent.reactor, "GET", "/_matrix/identity/v2/hash_details" ) request.requestHeaders.addRawHeader( b"Authorization", b"Bearer " + self.test_token.encode("ascii") ) token = tokenFromRequest(request) self.assertEqual(token, self.test_token) def test_can_read_token_from_query_parameters(self): """Tests that Sydent correctly extracts an auth token from query parameters""" self.sydent.run() request, _ = make_request( self.sydent.reactor, "GET", "/_matrix/identity/v2/hash_details?access_token=" + self.test_token, ) token = tokenFromRequest(request) self.assertEqual(token, self.test_token) sydent-2.5.1/tests/test_blacklisting.py000066400000000000000000000166631414516477000202730ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from unittest.mock import patch from twisted.internet.error import DNSLookupError from twisted.test.proto_helpers import StringTransport from twisted.trial.unittest import TestCase from twisted.web.client import Agent from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper from sydent.http.srvresolver import Server from tests.utils import AsyncMock, make_request, make_sydent class BlacklistingAgentTest(TestCase): def setUp(self): config = { "general": { "ip.blacklist": "5.0.0.0/8", "ip.whitelist": "5.1.1.1", }, } self.sydent = make_sydent(test_config=config) self.reactor = self.sydent.reactor self.safe_domain, self.safe_ip = b"safe.test", b"1.2.3.4" self.unsafe_domain, self.unsafe_ip = b"danger.test", b"5.6.7.8" self.allowed_domain, self.allowed_ip = b"allowed.test", b"5.1.1.1" # Configure the reactor's DNS resolver. for (domain, ip) in ( (self.safe_domain, self.safe_ip), (self.unsafe_domain, self.unsafe_ip), (self.allowed_domain, self.allowed_ip), ): self.reactor.lookups[domain.decode()] = ip.decode() self.reactor.lookups[ip.decode()] = ip.decode() self.ip_whitelist = self.sydent.config.general.ip_whitelist self.ip_blacklist = self.sydent.config.general.ip_blacklist def test_reactor(self): """Apply the blacklisting reactor and ensure it properly blocks connections to particular domains and IPs. """ agent = Agent( BlacklistingReactorWrapper( self.reactor, ip_whitelist=self.ip_whitelist, ip_blacklist=self.ip_blacklist, ), ) # The unsafe domains and IPs should be rejected. for domain in (self.unsafe_domain, self.unsafe_ip): self.failureResultOf( agent.request(b"GET", b"http://" + domain), DNSLookupError ) self.reactor.tcpClients = [] # The safe domains IPs should be accepted. for domain in ( self.safe_domain, self.allowed_domain, self.safe_ip, self.allowed_ip, ): agent.request(b"GET", b"http://" + domain) # Grab the latest TCP connection. ( host, port, client_factory, _timeout, _bindAddress, ) = self.reactor.tcpClients.pop() @patch( "sydent.http.srvresolver.SrvResolver.resolve_service", new_callable=AsyncMock ) def test_federation_client_allowed_ip(self, resolver): self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", { "access_token": "foo", "expires_in": 300, "matrix_server_name": "example.com", "token_type": "Bearer", }, ) resolver.return_value = [ Server( host=self.allowed_domain, port=443, priority=1, weight=1, expires=100, ) ] request.render(self.sydent.servlets.registerServlet) transport, protocol = self._get_http_request( self.allowed_ip.decode("ascii"), 443 ) self.assertRegex( transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo" ) self.assertRegex(transport.value(), b"Host: example.com") # Send it the HTTP response res_json = b'{ "sub": "@test:example.com" }' protocol.dataReceived( b"HTTP/1.1 200 OK\r\n" b"Server: Fake\r\n" b"Content-Type: application/json\r\n" b"Content-Length: %i\r\n" b"\r\n" b"%s" % (len(res_json), res_json) ) self.assertEqual(channel.code, 200) @patch( "sydent.http.srvresolver.SrvResolver.resolve_service", new_callable=AsyncMock ) def test_federation_client_safe_ip(self, resolver): self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", { "access_token": "foo", "expires_in": 300, "matrix_server_name": "example.com", "token_type": "Bearer", }, ) resolver.return_value = [ Server( host=self.safe_domain, port=443, priority=1, weight=1, expires=100, ) ] request.render(self.sydent.servlets.registerServlet) transport, protocol = self._get_http_request(self.safe_ip.decode("ascii"), 443) self.assertRegex( transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo" ) self.assertRegex(transport.value(), b"Host: example.com") # Send it the HTTP response res_json = b'{ "sub": "@test:example.com" }' protocol.dataReceived( b"HTTP/1.1 200 OK\r\n" b"Server: Fake\r\n" b"Content-Type: application/json\r\n" b"Content-Length: %i\r\n" b"\r\n" b"%s" % (len(res_json), res_json) ) self.assertEqual(channel.code, 200) @patch("sydent.http.srvresolver.SrvResolver.resolve_service") def test_federation_client_unsafe_ip(self, resolver): self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", { "access_token": "foo", "expires_in": 300, "matrix_server_name": "example.com", "token_type": "Bearer", }, ) resolver.return_value = [ Server( host=self.unsafe_domain, port=443, priority=1, weight=1, expires=100, ) ] request.render(self.sydent.servlets.registerServlet) self.assertNot(self.reactor.tcpClients) self.assertEqual(channel.code, 500) def _get_http_request(self, expected_host, expected_port): clients = self.reactor.tcpClients (host, port, factory, _timeout, _bindAddress) = clients[-1] self.assertEqual(host, expected_host) self.assertEqual(port, expected_port) # complete the connection and wire it up to a fake transport protocol = factory.buildProtocol(None) transport = StringTransport() protocol.makeConnection(transport) return transport, protocol sydent-2.5.1/tests/test_casefold_migration.py000066400000000000000000000256441414516477000214550ustar00rootroot00000000000000import json import os.path from unittest.mock import patch from twisted.trial import unittest from scripts.casefold_db import ( calculate_lookup_hash, update_global_associations, update_local_associations, ) from sydent.util import json_decoder from sydent.util.emailutils import sendEmail from tests.utils import make_sydent class MigrationTestCase(unittest.TestCase): def create_signedassoc(self, medium, address, mxid, ts, not_before, not_after): return { "medium": medium, "address": address, "mxid": mxid, "ts": ts, "not_before": not_before, "not_after": not_after, } def setUp(self): # Create a new sydent config = { "general": { "templates.path": os.path.join( os.path.dirname(os.path.dirname(__file__)), "res" ), }, "crypto": { "ed25519.signingkey": "ed25519 0 FJi1Rnpj3/otydngacrwddFvwz/dTDsBv62uZDN2fZM" }, } self.sydent = make_sydent(test_config=config) # create some local associations associations = [] for i in range(10): address = "bob%d@example.com" % i associations.append( { "medium": "email", "address": address, "lookup_hash": calculate_lookup_hash(self.sydent, address), "mxid": "@bob%d:example.com" % i, "ts": (i * 10000), "not_before": 0, "not_after": 99999999999, } ) # create some casefold-conflicting associations for i in range(5): address = "BOB%d@example.com" % i associations.append( { "medium": "email", "address": address, "lookup_hash": calculate_lookup_hash(self.sydent, address), "mxid": "@otherbob%d:example.com" % i, "ts": (i * 10000), "not_before": 0, "not_after": 99999999999, } ) # add all associations to db cur = self.sydent.db.cursor() cur.executemany( "INSERT INTO local_threepid_associations " "(medium, address, lookup_hash, mxid, ts, notBefore, notAfter) " "VALUES (?, ?, ?, ?, ?, ?, ?)", [ ( assoc["medium"], assoc["address"], assoc["lookup_hash"], assoc["mxid"], assoc["ts"], assoc["not_before"], assoc["not_after"], ) for assoc in associations ], ) self.sydent.db.commit() # create some global associations associations = [] originServer = self.sydent.config.general.server_name for i in range(10): address = "bob%d@example.com" % i mxid = "@bob%d:example.com" % i ts = 10000 * i associations.append( { "medium": "email", "address": address, "lookup_hash": calculate_lookup_hash(self.sydent, address), "mxid": mxid, "ts": ts, "not_before": 0, "not_after": 99999999999, "originServer": originServer, "originId": i, "sgAssoc": json.dumps( self.create_signedassoc( "email", address, mxid, ts, 0, 99999999999 ) ), } ) # create some casefold-conflicting associations for i in range(5): address = "BOB%d@example.com" % i mxid = "@BOB%d:example.com" % i ts = 10000 * i associations.append( { "medium": "email", "address": address, "lookup_hash": calculate_lookup_hash(self.sydent, address), "mxid": mxid, "ts": ts + 1, "not_before": 0, "not_after": 99999999999, "originServer": originServer, "originId": i + 10, "sgAssoc": json.dumps( self.create_signedassoc( "email", address, mxid, ts, 0, 99999999999 ) ), } ) # add all associations to db cur = self.sydent.db.cursor() cur.executemany( "INSERT INTO global_threepid_associations " "(medium, address, lookup_hash, mxid, ts, notBefore, notAfter, originServer, originId, sgAssoc) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [ ( assoc["medium"], assoc["address"], assoc["lookup_hash"], assoc["mxid"], assoc["ts"], assoc["not_before"], assoc["not_after"], assoc["originServer"], assoc["originId"], assoc["sgAssoc"], ) for assoc in associations ], ) self.sydent.db.commit() def test_migration_email(self): with patch("sydent.util.emailutils.smtplib") as smtplib: # self.sydent.config.email.template is deprecated if self.sydent.config.email.template is None: templateFile = self.sydent.get_branded_template( None, "migration_template.eml", ) else: templateFile = self.sydent.config.email.template sendEmail( self.sydent, templateFile, "bob@example.com", { "mxid": "@bob:example.com", "subject_header_value": "MatrixID Deletion", }, ) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertIn("In the past", email_contents) # test email was sent smtp.sendmail.assert_called() def test_local_db_migration(self): with patch("sydent.util.emailutils.smtplib") as smtplib: update_local_associations( self.sydent, self.sydent.db, send_email=True, dry_run=False, test=True, ) # test 5 emails were sent smtp = smtplib.SMTP.return_value self.assertEqual(smtp.sendmail.call_count, 5) # don't send emails to people who weren't affected self.assertNotIn( smtp.sendmail.call_args_list, [ "bob5@example.com", "bob6@example.com", "bob7@example.com", "bob8@example.com", "bob9@example.com", ], ) # make sure someone who is affected gets email self.assertIn("bob4@example.com", smtp.sendmail.call_args_list[0][0]) cur = self.sydent.db.cursor() res = cur.execute("SELECT * FROM local_threepid_associations") db_state = res.fetchall() # five addresses should have been deleted self.assertEqual(len(db_state), 10) # iterate through db and make sure all addresses are casefolded and hash matches casefolded address for row in db_state: casefolded = row[2].casefold() self.assertEqual(row[2], casefolded) self.assertEqual( calculate_lookup_hash(self.sydent, row[2]), calculate_lookup_hash(self.sydent, casefolded), ) def test_global_db_migration(self): update_global_associations( self.sydent, self.sydent.db, dry_run=False, ) cur = self.sydent.db.cursor() res = cur.execute("SELECT * FROM global_threepid_associations") db_state = res.fetchall() # five addresses should have been deleted self.assertEqual(len(db_state), 10) # iterate through db and make sure all addresses are casefolded and hash matches casefolded address # and make sure the casefolded address matches the address in sgAssoc for row in db_state: casefolded = row[2].casefold() self.assertEqual(row[2], casefolded) self.assertEqual( calculate_lookup_hash(self.sydent, row[2]), calculate_lookup_hash(self.sydent, casefolded), ) sgassoc = json_decoder.decode(row[9]) self.assertEqual(row[2], sgassoc["address"]) def test_local_no_email_does_not_send_email(self): with patch("sydent.util.emailutils.smtplib") as smtplib: update_local_associations( self.sydent, self.sydent.db, send_email=False, dry_run=False, test=True, ) smtp = smtplib.SMTP.return_value # test no emails were sent self.assertEqual(smtp.sendmail.call_count, 0) def test_dry_run_does_nothing(self): # reset DB self.setUp() cur = self.sydent.db.cursor() # grab a snapshot of global table before running script res1 = cur.execute("SELECT mxid FROM global_threepid_associations") list1 = res1.fetchall() with patch("sydent.util.emailutils.smtplib") as smtplib: update_global_associations( self.sydent, self.sydent.db, dry_run=True, ) # test no emails were sent smtp = smtplib.SMTP.return_value self.assertEqual(smtp.sendmail.call_count, 0) res2 = cur.execute("SELECT mxid FROM global_threepid_associations") list2 = res2.fetchall() self.assertEqual(list1, list2) # grab a snapshot of local table db before running script res3 = cur.execute("SELECT mxid FROM local_threepid_associations") list3 = res3.fetchall() with patch("sydent.util.emailutils.smtplib") as smtplib: update_local_associations( self.sydent, self.sydent.db, send_email=True, dry_run=True, test=True, ) # test no emails were sent smtp = smtplib.SMTP.return_value self.assertEqual(smtp.sendmail.call_count, 0) res4 = cur.execute("SELECT mxid FROM local_threepid_associations") list4 = res4.fetchall() self.assertEqual(list3, list4) sydent-2.5.1/tests/test_email.py000066400000000000000000000066371414516477000167140ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path from unittest.mock import patch from twisted.trial import unittest from tests.utils import make_request, make_sydent class TestRequestCode(unittest.TestCase): def setUp(self): # Create a new sydent config = { "general": { "templates.path": os.path.join( os.path.dirname(os.path.dirname(__file__)), "res" ), }, } self.sydent = make_sydent(test_config=config) def _render_request(self, request): # Patch out the email sending so we can investigate the resulting email. with patch("sydent.util.emailutils.smtplib") as smtplib: request.render(self.sydent.servlets.emailRequestCode) # Fish out the SMTP object and return it. smtp = smtplib.SMTP.return_value smtp.sendmail.assert_called_once() return smtp def test_request_code(self): self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/api/v1/validate/email/requestToken", { "email": "test@test", "client_secret": "oursecret", "send_attempt": 0, }, ) smtp = self._render_request(request) self.assertEqual(channel.code, 200) # Ensure the email is as expected. email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertIn("Confirm your email address for Matrix", email_contents) def test_request_code_via_url_query_params(self): self.sydent.run() url = ( "/_matrix/identity/api/v1/validate/email/requestToken?" "email=test@test" "&client_secret=oursecret" "&send_attempt=0" ) request, channel = make_request(self.sydent.reactor, "POST", url) smtp = self._render_request(request) self.assertEqual(channel.code, 200) # Ensure the email is as expected. email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertIn("Confirm your email address for Matrix", email_contents) def test_branded_request_code(self): self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/api/v1/validate/email/requestToken?brand=vector-im", { "email": "test@test", "client_secret": "oursecret", "send_attempt": 0, }, ) smtp = self._render_request(request) self.assertEqual(channel.code, 200) # Ensure the email is as expected. email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertIn("Confirm your email address for Element", email_contents) sydent-2.5.1/tests/test_invites.py000066400000000000000000000131131414516477000172710ustar00rootroot00000000000000from unittest.mock import Mock from twisted.trial import unittest from twisted.web.client import Response from sydent.db.invite_tokens import JoinTokenStore from sydent.http.httpclient import FederationHttpClient from sydent.http.servlets.store_invite_servlet import StoreInviteServlet from tests.utils import make_sydent class ThreepidInvitesTestCase(unittest.TestCase): """Tests features related to storing and delivering 3PID invites.""" def setUp(self): # Create a new sydent config = { "email": { # Used by test_invited_email_address_obfuscation "email.third_party_invite_username_obfuscate_characters": "6", "email.third_party_invite_domain_obfuscate_characters": "8", }, } self.sydent = make_sydent(test_config=config) def test_delete_on_bind(self): """Tests that 3PID invite tokens are deleted upon delivery after a successful bind. """ self.sydent.run() # The 3PID we're working with. medium = "email" address = "john@example.com" # Mock post_json_get_nothing so the /onBind call doesn't fail. async def post_json_get_nothing(uri, post_json, opts): return Response((b"HTTP", 1, 1), 200, b"OK", None, None) FederationHttpClient.post_json_get_nothing = Mock( side_effect=post_json_get_nothing, ) # Manually insert an invite token, we'll check later that it's been deleted. join_token_store = JoinTokenStore(self.sydent) join_token_store.storeToken( medium, address, "!someroom:example.com", "@jane:example.com", "sometoken", ) # Make sure the token still exists and can be retrieved. tokens = join_token_store.getTokens(medium, address) self.assertEqual(len(tokens), 1, tokens) # Bind the 3PID self.sydent.threepidBinder.addBinding( medium, address, "@john:example.com", ) # Give Sydent some time to call /onBind and delete the token. self.sydent.reactor.advance(1000) cur = self.sydent.db.cursor() # Manually retrieve the tokens for this 3PID. We don't use getTokens because it # filters out sent tokens, so would return nothing even if the token hasn't been # deleted. res = cur.execute( "SELECT medium, address, room_id, sender, token FROM invite_tokens" " WHERE medium = ? AND address = ?", ( medium, address, ), ) rows = res.fetchall() # Check that we didn't get any result. self.assertEqual(len(rows), 0, rows) def test_invited_email_address_obfuscation(self): """Test that email addresses included in third-party invites are properly obfuscated according to the relevant config options """ store_invite_servlet = StoreInviteServlet(self.sydent) email_address = "1234567890@1234567890.com" redacted_address = store_invite_servlet.redact_email_address(email_address) self.assertEqual(redacted_address, "123456...@12345678...") # Even short addresses are redacted short_email_address = "1@1.com" redacted_address = store_invite_servlet.redact_email_address( short_email_address ) self.assertEqual(redacted_address, "...@1...") class ThreepidInvitesNoDeleteTestCase(unittest.TestCase): """Test that invite tokens are not deleted when that is disabled.""" def setUp(self): # Create a new sydent config = {"general": {"delete_tokens_on_bind": "false"}} self.sydent = make_sydent(test_config=config) def test_no_delete_on_bind(self): self.sydent.run() # The 3PID we're working with. medium = "email" address = "john@example.com" # Mock post_json_get_nothing so the /onBind call doesn't fail. async def post_json_get_nothing(uri, post_json, opts): return Response((b"HTTP", 1, 1), 200, b"OK", None, None) FederationHttpClient.post_json_get_nothing = Mock( side_effect=post_json_get_nothing, ) # Manually insert an invite token, we'll check later that it's been deleted. join_token_store = JoinTokenStore(self.sydent) join_token_store.storeToken( medium, address, "!someroom:example.com", "@jane:example.com", "sometoken", ) # Make sure the token still exists and can be retrieved. tokens = join_token_store.getTokens(medium, address) self.assertEqual(len(tokens), 1, tokens) # Bind the 3PID self.sydent.threepidBinder.addBinding( medium, address, "@john:example.com", ) # Give Sydent some time to call /onBind and delete the token. self.sydent.reactor.advance(1000) cur = self.sydent.db.cursor() # Manually retrieve the tokens for this 3PID. We don't use getTokens because it # filters out sent tokens, so would return nothing even if the token hasn't been # deleted. res = cur.execute( "SELECT medium, address, room_id, sender, token FROM invite_tokens" " WHERE medium = ? AND address = ?", ( medium, address, ), ) rows = res.fetchall() # Check that we didn't get any result. self.assertEqual(len(rows), 1, rows) sydent-2.5.1/tests/test_jinja_templates.py000066400000000000000000000207551414516477000207730ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path import urllib from unittest.mock import Mock, patch from twisted.trial import unittest from sydent.util.emailutils import sendEmail from tests.utils import make_sydent class TestTemplate(unittest.TestCase): def setUp(self): # Create a new sydent config = { "general": { "templates.path": os.path.join( os.path.dirname(os.path.dirname(__file__)), "res" ), }, } self.sydent = make_sydent(test_config=config) def test_jinja_vector_invite(self): substitutions = { "address": "foo@example.com", "medium": "email", "room_alias": "#somewhere:exmaple.org", "room_avatar_url": "mxc://example.org/s0meM3dia", "room_id": "!something:example.org", "room_name": "Bob's Emporium of Messages", "sender": "@bob:example.com", "sender_avatar_url": "mxc://example.org/an0th3rM3dia", "sender_display_name": "", "bracketed_verified_sender": "Bob Smith", "bracketed_room_name": "Bob's Emporium of Messages", "to": "person@test.test", "token": "a_token", "ephemeral_private_key": "mystery_key", "web_client_location": "https://app.element.io", "room_type": "", } # self.sydent.config.email.invite_template is deprecated if self.sydent.config.email.invite_template is None: templateFile = self.sydent.get_branded_template( "vector-im", "invite_template.eml", ) else: templateFile = self.sydent.config.email.invite_template with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") # test url input is encoded self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) # test html input is escaped self.assertIn("Bob's Emporium of Messages", email_contents) # test safe values are not escaped self.assertIn("", email_contents) # test our link is as expected expected_url = ( "https://app.element.io/#/room/" + urllib.parse.quote("!something:example.org") + "?email=" + urllib.parse.quote("test@test.com") + "&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D" + urllib.parse.quote("a_token") + "%26private_key%3D" + urllib.parse.quote("mystery_key") + "&room_name=" + urllib.parse.quote("Bob's Emporium of Messages") + "&room_avatar_url=" + urllib.parse.quote("mxc://example.org/s0meM3dia") + "&inviter_name=" + urllib.parse.quote("") + "&guest_access_token=&guest_user_id=&room_type=" ) text = email_contents.splitlines() link = text[19] self.assertEqual(link, expected_url) def test_jinja_matrix_invite(self): substitutions = { "address": "foo@example.com", "medium": "email", "room_alias": "#somewhere:exmaple.org", "room_avatar_url": "mxc://example.org/s0meM3dia", "room_id": "!something:example.org", "room_name": "Bob's Emporium of Messages", "sender": "@bob:example.com", "sender_avatar_url": "mxc://example.org/an0th3rM3dia", "sender_display_name": "", "bracketed_verified_sender": "Bob Smith", "bracketed_room_name": "Bob's Emporium of Messages", "to": "person@test.test", "token": "a_token", "ephemeral_private_key": "mystery_key", "web_client_location": "https://matrix.org", "room_type": "", } # self.sydent.config.email.invite_template is deprecated if self.sydent.config.email.invite_template is None: templateFile = self.sydent.get_branded_template( "matrix-org", "invite_template.eml", ) else: templateFile = self.sydent.config.email.invite_template with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") # test url input is encoded self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) # test html input is escaped self.assertIn("Bob's Emporium of Messages", email_contents) # test safe values are not escaped self.assertIn("", email_contents) # test our link is as expected expected_url = ( "https://matrix.org/#/room/" + urllib.parse.quote("!something:example.org") + "?email=" + urllib.parse.quote("test@test.com") + "&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D" + urllib.parse.quote("a_token") + "%26private_key%3D" + urllib.parse.quote("mystery_key") + "&room_name=" + urllib.parse.quote("Bob's Emporium of Messages") + "&room_avatar_url=" + urllib.parse.quote("mxc://example.org/s0meM3dia") + "&inviter_name=" + urllib.parse.quote("") + "&guest_access_token=&guest_user_id=&room_type=" ) text = email_contents.splitlines() link = text[22] self.assertEqual(link, expected_url) def test_jinja_matrix_verification(self): substitutions = { "address": "foo@example.com", "medium": "email", "to": "person@test.test", "token": "<>", "link": "https://link_test.com", } templateFile = self.sydent.get_branded_template( "matrix-org", "verification_template.eml", ) with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") # test html input is escaped self.assertIn("<<token>>", email_contents) # test safe values are not escaped self.assertIn("<>", email_contents) @patch( "sydent.util.emailutils.generateAlphanumericTokenOfLength", Mock(return_value="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), ) def test_jinja_vector_verification(self): substitutions = { "address": "foo@example.com", "medium": "email", "to": "person@test.test", "link": "https://link_test.com", } templateFile = self.sydent.get_branded_template( "vector-im", "verification_template.eml", ) with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") path = os.path.join( self.sydent.config.general.templates_path, "vector_verification_sample.txt", ) with open(path, "r") as file: expected_text = file.read() # remove the email headers as they are variable email_contents = email_contents[email_contents.index("Hello") :] # test all ouput is as expected self.assertEqual(email_contents, expected_text) sydent-2.5.1/tests/test_msisdn.py000066400000000000000000000054621414516477000171150ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import asyncio import os.path from typing import Awaitable from unittest.mock import MagicMock, patch from twisted.trial import unittest from twisted.web.server import Request from tests.utils import make_request, make_sydent class TestRequestCode(unittest.TestCase): def setUp(self) -> None: # Create a new sydent config = { "general": { "templates.path": os.path.join( os.path.dirname(os.path.dirname(__file__)), "res" ), }, } self.sydent = make_sydent(test_config=config) def _render_request(self, request: Request) -> Awaitable[MagicMock]: # Patch out the email sending so we can investigate the resulting email. with patch("sydent.sms.openmarket.OpenMarketSMS.sendTextSMS") as sendTextSMS: # We can't use AsyncMock until Python 3.8. Instead, mock the # function as returning a future. f = asyncio.Future() f.set_result(MagicMock()) sendTextSMS.return_value = f request.render(self.sydent.servlets.msisdnRequestCode) return sendTextSMS def test_request_code(self) -> None: self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/api/v1/validate/msisdn/requestToken", { "phone_number": "447700900750", "country": "GB", "client_secret": "oursecret", "send_attempt": 0, }, ) sendSMS_mock = self._render_request(request) sendSMS_mock.assert_called_once() self.assertEqual(channel.code, 200) def test_request_code_via_url_query_params(self) -> None: self.sydent.run() url = ( "/_matrix/identity/api/v1/validate/msisdn/requestToken?" "phone_number=447700900750" "&country=GB" "&client_secret=oursecret" "&send_attempt=0" ) request, channel = make_request(self.sydent.reactor, "POST", url) sendSMS_mock = self._render_request(request) sendSMS_mock.assert_called_once() self.assertEqual(channel.code, 200) sydent-2.5.1/tests/test_register.py000066400000000000000000000076251414516477000174470ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from http import HTTPStatus from json import JSONDecodeError from unittest.mock import patch import twisted.internet.error import twisted.web.client from parameterized import parameterized from twisted.trial import unittest from tests.utils import make_request, make_sydent class RegisterTestCase(unittest.TestCase): """Tests Sydent's register servlet""" def setUp(self) -> None: # Create a new sydent self.sydent = make_sydent() def test_sydent_rejects_invalid_hostname(self) -> None: """Tests that the /register endpoint rejects an invalid hostname passed as matrix_server_name""" self.sydent.run() bad_hostname = "example.com#" request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", content={"matrix_server_name": bad_hostname, "access_token": "foo"}, ) request.render(self.sydent.servlets.registerServlet) self.assertEqual(channel.code, 400) @parameterized.expand( [ (twisted.internet.error.DNSLookupError(),), (twisted.internet.error.TimeoutError(),), (twisted.internet.error.ConnectionRefusedError(),), # Naughty: strictly we're supposed to initialise a ResponseNeverReceived # with a list of 1 or more failures. (twisted.web.client.ResponseNeverReceived([]),), ] ) def test_connection_failure(self, exc: Exception) -> None: self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", content={ "matrix_server_name": "matrix.alice.com", "access_token": "back_in_wonderland", }, ) servlet = self.sydent.servlets.registerServlet with patch.object(servlet.client, "get_json", side_effect=exc): request.render(servlet) self.assertEqual(channel.code, HTTPStatus.INTERNAL_SERVER_ERROR) self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN") # Check that we haven't just returned the generic error message in asyncjsonwrap self.assertNotEqual(channel.json_body["error"], "Internal Server Error") self.assertIn("contact", channel.json_body["error"]) def test_federation_does_not_return_json(self) -> None: self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/register", content={ "matrix_server_name": "matrix.alice.com", "access_token": "back_in_wonderland", }, ) servlet = self.sydent.servlets.registerServlet exc = JSONDecodeError("ruh roh", "C'est n'est pas une objet JSON", 0) with patch.object(servlet.client, "get_json", side_effect=exc): request.render(servlet) self.assertEqual(channel.code, HTTPStatus.INTERNAL_SERVER_ERROR) self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN") # Check that we haven't just returned the generic error message in asyncjsonwrap self.assertNotEqual(channel.json_body["error"], "Internal Server Error") self.assertIn("JSON", channel.json_body["error"]) sydent-2.5.1/tests/test_replication.py000066400000000000000000000167201414516477000201300ustar00rootroot00000000000000import json from unittest.mock import Mock from twisted.internet import defer from twisted.trial import unittest from twisted.web.client import Response from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer from tests.utils import make_request, make_sydent class ReplicationTestCase(unittest.TestCase): """Test that a Sydent can correctly replicate data with another Sydent""" def setUp(self): # Create a new sydent config = { "crypto": { "ed25519.signingkey": "ed25519 0 FJi1Rnpj3/otydngacrwddFvwz/dTDsBv62uZDN2fZM" } } self.sydent = make_sydent(test_config=config) # Create a fake peer to replicate to. peer_public_key_base64 = "+vB8mTaooD/MA8YYZM8t9+vnGhP1937q2icrqPV9JTs" # Inject our fake peer into the database. cur = self.sydent.db.cursor() cur.execute( "INSERT INTO peers (name, port, lastSentVersion, active) VALUES (?, ?, ?, ?)", ("fake.server", 1234, 0, 1), ) cur.execute( "INSERT INTO peer_pubkeys (peername, alg, key) VALUES (?, ?, ?)", ("fake.server", "ed25519", peer_public_key_base64), ) self.sydent.db.commit() # Build some fake associations. self.assocs = [] assoc_count = 150 for i in range(assoc_count): assoc = ThreepidAssociation( medium="email", address="bob%d@example.com" % i, lookup_hash=None, mxid="@bob%d:example.com" % i, ts=(i * 10000), not_before=0, not_after=99999999999, ) self.assocs.append(assoc) def test_incoming_replication(self): """Impersonate a peer that sends a replication push to Sydent, then checks that it accepts the payload and saves it correctly. """ self.sydent.run() # Configure the Sydent to impersonate. We need to use "fake.server" as the # server's name because that's the name the recipient Sydent has for it. On top # of that, the replication servlet expects a TLS certificate in the request so it # can extract a common name and figure out which peer sent it from its common # name. The common name of the certificate we use for tests is fake.server. config = { "general": {"server.name": "fake.server"}, "crypto": { "ed25519.signingkey": "ed25519 0 b29eXMMAYCFvFEtq9mLI42aivMtcg4Hl0wK89a+Vb6c" }, } fake_sender_sydent = make_sydent(config) signer = Signer(fake_sender_sydent) # Sign the associations with the Sydent to impersonate so the recipient Sydent # can verify the signatures on them. signed_assocs = {} for assoc_id, assoc in enumerate(self.assocs): signed_assoc = signer.signedThreePidAssociation(assoc) signed_assocs[assoc_id] = signed_assoc # Send the replication push. body = json.dumps({"sgAssocs": signed_assocs}) request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/replicate/v1/push", body ) request.render(self.sydent.servlets.replicationPush) self.assertEqual(channel.code, 200) # Check that the recipient Sydent has correctly saved the associations in the # push. cur = self.sydent.db.cursor() res = cur.execute("SELECT originId, sgAssoc FROM global_threepid_associations") res_assocs = {} for row in res.fetchall(): originId = row[0] signed_assoc = json.loads(row[1]) res_assocs[originId] = signed_assoc for assoc_id, signed_assoc in signed_assocs.items(): self.assertDictEqual(signed_assoc, res_assocs[assoc_id]) def test_outgoing_replication(self): """Make a fake peer and associations and make sure Sydent tries to push to it.""" cur = self.sydent.db.cursor() # Insert the fake associations into the database. cur.executemany( "INSERT INTO local_threepid_associations " "(medium, address, lookup_hash, mxid, ts, notBefore, notAfter) " "VALUES (?, ?, ?, ?, ?, ?, ?)", [ ( assoc.medium, assoc.address, assoc.lookup_hash, assoc.mxid, assoc.ts, assoc.not_before, assoc.not_after, ) for assoc in self.assocs ], ) self.sydent.db.commit() # Manually sign all associations so we can check whether Sydent attempted to # push the same. signer = Signer(self.sydent) signed_assocs = {} for assoc_id, assoc in enumerate(self.assocs): signed_assoc = signer.signedThreePidAssociation(assoc) signed_assocs[assoc_id] = signed_assoc sent_assocs = {} def request(method, uri, headers, body): """ Processes a request sent to the mocked agent. :param method: The method of the request. :type method: bytes :param uri: The URI of the request. :type uri: bytes :param headers: The headers of the request. :type headers: twisted.web.http_headers.Headers :param body: The body of the request. :type body: twisted.web.client.FileBodyProducer[io.BytesIO] :return: A deferred that resolves into a 200 OK response. :rtype: twisted.internet.defer.Deferred[Response] """ # Check the method and the URI. assert method == b"POST" assert uri == b"https://fake.server:1234/_matrix/identity/replicate/v1/push" # postJson calls the agent with a BytesIO within a FileBodyProducer, so we # need to unpack the payload correctly. payload = json.loads(body._inputFile.read().decode("utf8")) for assoc_id, assoc in payload["sgAssocs"].items(): sent_assocs[assoc_id] = assoc # Return with a fake response wrapped in a Deferred. d = defer.Deferred() d.callback(Response((b"HTTP", 1, 1), 200, b"OK", None, None)) return d # Mock the replication client's agent so it runs the custom code instead of # actually sending the requests. agent = Mock(spec=["request"]) agent.request.side_effect = request self.sydent.replicationHttpsClient.agent = agent # Start Sydent and allow some time for all the necessary pushes to happen. self.sydent.run() self.sydent.reactor.advance(1000) # Check that, now that Sydent pushed all the associations it was meant to, we # have all of the associations we initially inserted. self.assertEqual(len(self.assocs), len(sent_assocs)) for assoc_id, assoc in sent_assocs.items(): # Replication payloads use a specific format that causes the JSON encoder to # convert the numeric indexes to string, so we need to convert them back when # looking up in signed_assocs. Also, the ID of the first association Sydent # will push will be 1, so we need to subtract 1 when figuring out which index # to lookup. self.assertDictEqual(assoc, signed_assocs[int(assoc_id) - 1]) sydent-2.5.1/tests/test_start.py000066400000000000000000000003631414516477000167500ustar00rootroot00000000000000from twisted.trial import unittest from tests.utils import make_sydent class StartupTestCase(unittest.TestCase): """Test that sydent started up correctly""" def test_start(self): sydent = make_sydent() sydent.run() sydent-2.5.1/tests/test_store_invite.py000066400000000000000000000044011414516477000203220ustar00rootroot00000000000000# Copyright 2021 Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path from unittest.mock import patch from parameterized import parameterized from twisted.trial import unittest from sydent.users.accounts import Account from tests.utils import make_request, make_sydent class StoreInviteTestCase(unittest.TestCase): """Tests Sydent's register servlet""" def setUp(self) -> None: # Create a new sydent config = { "general": { "templates.path": os.path.join( os.path.dirname(os.path.dirname(__file__)), "res" ), }, "email": { "email.from": "Sydent Validation ", }, } self.sydent = make_sydent(test_config=config) self.sender = "@alice:wonderland" @parameterized.expand( [ ("not@an@email@address",), ("Naughty Nigel ",), ] ) def test_invalid_email_returns_400(self, address: str) -> None: self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/account/store-invite", content={ "address": address, "medium": "email", "room_id": "!myroom:test", "sender": self.sender, }, ) with patch("sydent.http.servlets.store_invite_servlet.authV2") as authV2: authV2.return_value = Account(self.sender, 0, None) request.render(self.sydent.servlets.storeInviteServletV2) self.assertEqual(channel.code, 400) self.assertEqual(channel.json_body["errcode"], "M_INVALID_EMAIL") sydent-2.5.1/tests/test_threepidunbind.py000066400000000000000000000046641414516477000206270ustar00rootroot00000000000000# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from http import HTTPStatus from unittest.mock import patch import twisted.internet.error import twisted.web.client from parameterized import parameterized from twisted.trial import unittest from tests.utils import make_request, make_sydent class ThreepidUnbindTestCase(unittest.TestCase): """Tests Sydent's threepidunbind servlet""" def setUp(self) -> None: # Create a new sydent self.sydent = make_sydent() # Duplicated from TestRegisterServelet. Is there a way for us to keep # ourselves DRY? @parameterized.expand( [ (twisted.internet.error.DNSLookupError(),), (twisted.internet.error.TimeoutError(),), (twisted.internet.error.ConnectionRefusedError(),), # Naughty: strictly we're supposed to initialise a ResponseNeverReceived # with a list of 1 or more failures. (twisted.web.client.ResponseNeverReceived([]),), ] ) def test_connection_failure(self, exc: Exception) -> None: """Check we respond sensibly if we can't contact the homeserver.""" self.sydent.run() request, channel = make_request( self.sydent.reactor, "POST", "/_matrix/identity/v2/3pid/unbind", content={ "mxid": "@alice:wonderland", "threepid": { "address": "alice.cooper@wonderland.biz", "medium": "email", }, }, ) with patch.object( self.sydent.sig_verifier, "authenticate_request", side_effect=exc ): request.render(self.sydent.servlets.threepidUnbind) self.assertEqual(channel.code, HTTPStatus.INTERNAL_SERVER_ERROR) self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN") self.assertIn("contact", channel.json_body["error"]) sydent-2.5.1/tests/test_util.py000066400000000000000000000037241414516477000165740ustar00rootroot00000000000000from twisted.trial import unittest from sydent.util.stringutils import is_valid_matrix_server_name class UtilTests(unittest.TestCase): """Tests Sydent utility functions.""" def test_is_valid_matrix_server_name(self): """Tests that the is_valid_matrix_server_name function accepts only valid hostnames (or domain names), with optional port number. """ self.assertTrue(is_valid_matrix_server_name("9.9.9.9")) self.assertTrue(is_valid_matrix_server_name("9.9.9.9:4242")) self.assertTrue(is_valid_matrix_server_name("[::]")) self.assertTrue(is_valid_matrix_server_name("[::]:4242")) self.assertTrue(is_valid_matrix_server_name("[a:b:c::]:4242")) self.assertTrue(is_valid_matrix_server_name("example.com")) self.assertTrue(is_valid_matrix_server_name("EXAMPLE.COM")) self.assertTrue(is_valid_matrix_server_name("ExAmPlE.CoM")) self.assertTrue(is_valid_matrix_server_name("example.com:4242")) self.assertTrue(is_valid_matrix_server_name("localhost")) self.assertTrue(is_valid_matrix_server_name("localhost:9000")) self.assertTrue(is_valid_matrix_server_name("a.b.c.d:1234")) self.assertFalse(is_valid_matrix_server_name("[:::]")) self.assertFalse(is_valid_matrix_server_name("a:b:c::")) self.assertFalse(is_valid_matrix_server_name("example.com:65536")) self.assertFalse(is_valid_matrix_server_name("example.com:0")) self.assertFalse(is_valid_matrix_server_name("example.com:-1")) self.assertFalse(is_valid_matrix_server_name("example.com:a")) self.assertFalse(is_valid_matrix_server_name("example.com: ")) self.assertFalse(is_valid_matrix_server_name("example.com:04242")) self.assertFalse(is_valid_matrix_server_name("example.com: 4242")) self.assertFalse(is_valid_matrix_server_name("example.com/example.com")) self.assertFalse(is_valid_matrix_server_name("example.com#example.com")) sydent-2.5.1/tests/utils.py000066400000000000000000000231331414516477000157140ustar00rootroot00000000000000import json import logging import os from io import BytesIO from typing import Dict from unittest.mock import MagicMock import attr import twisted.logger from OpenSSL import crypto from twisted.internet import address from twisted.internet._resolver import SimpleResolverComplexifier from twisted.internet.defer import fail, succeed from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import ( IHostnameResolver, IReactorPluggableNameResolver, IResolverSimple, ) from twisted.test.proto_helpers import MemoryReactorClock from twisted.web.http import unquote from twisted.web.http_headers import Headers from twisted.web.server import Request, Site from zope.interface import implementer from sydent.config import SydentConfig from sydent.sydent import Sydent # Expires on Jan 11 2030 at 17:53:40 GMT FAKE_SERVER_CERT_PEM = """ -----BEGIN CERTIFICATE----- MIIDlzCCAn+gAwIBAgIUC8tnJVZ8Cawh5tqr7PCAOfvyGTYwDQYJKoZIhvcNAQEL BQAwWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLZmFrZS5zZXJ2ZXIw HhcNMjAwMTE0MTc1MzQwWhcNMzAwMTExMTc1MzQwWjBbMQswCQYDVQQGEwJBVTET MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ dHkgTHRkMRQwEgYDVQQDDAtmYWtlLnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQAD ggEPADCCAQoCggEBANNzY7YHBLm4uj52ojQc/dfQCoR+63IgjxZ6QdnThhIlOYgE 3y0Ks49bt3GKmAweOFRRKfDhJRKCYfqZTYudMcdsQg696s2HhiTY0SpqO0soXwW4 6kEIxnTy2TqkPjWlsWgGTtbVnKc5pnLs7MaQwLIQfxirqD2znn+9r68WMOJRlzkv VmrXDXjxKPANJJ9b0PiGrL2SF4QcF3zHk8Tjf24OGRX4JTNwiGraU/VN9rrqSHug CLWcfZ1mvcav3scvtGfgm4kxcw8K6heiQAc3QAMWIrdWhiunaWpQYgw7euS8lZ/O C7HZ7YbdoldknWdK8o7HJZmxUP9yW9Pqa3n8p9UCAwEAAaNTMFEwHQYDVR0OBBYE FHwfTq0Mdk9YKqjyfdYm4v9zRP8nMB8GA1UdIwQYMBaAFHwfTq0Mdk9YKqjyfdYm 4v9zRP8nMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEPVM5/+ Sj9P/CvNG7F2PxlDQC1/+aVl6ARAz/bZmm7yJnWEleBSwwFLerEQU6KFrgjA243L qgY6Qf2EYUn1O9jroDg/IumlcQU1H4DXZ03YLKS2bXFGj630Piao547/l4/PaKOP wSvwDcJlBatKfwjMVl3Al/EcAgUJL8eVosnqHDSINdBuFEc8Kw4LnDSFoTEIx19i c+DKmtnJNI68wNydLJ3lhSaj4pmsX4PsRqsRzw+jgkPXIG1oGlUDMO3k7UwxfYKR XkU5mFYkohPTgxv5oYGq2FCOPixkbov7geCEvEUs8m8c8MAm4ErBUzemOAj8KVhE tWVEpHfT+G7AjA8= -----END CERTIFICATE----- """ def make_sydent(test_config={}): """Create a new sydent Args: test_config (dict): any configuration variables for overriding the default sydent config """ # Use an in-memory SQLite database. Note that the database isn't cleaned up between # tests, so by default the same database will be used for each test if changed to be # a file on disk. if "db" not in test_config: test_config["db"] = {"db.file": ":memory:"} else: test_config["db"].setdefault("db.file", ":memory:") reactor = ResolvingMemoryReactorClock() sydent_config = SydentConfig() sydent_config.parse_config_dict(test_config) return Sydent( reactor=reactor, sydent_config=sydent_config, use_tls_for_federation=False, ) @attr.s class FakeChannel: """ A fake Twisted Web Channel (the part that interfaces with the wire). Mostly copied from Synapse's tests framework. """ site = attr.ib(type=Site) _reactor = attr.ib() result = attr.ib(default=attr.Factory(dict)) _producer = None @property def json_body(self): if not self.result: raise Exception("No result yet.") return json.loads(self.result["body"].decode("utf8")) @property def code(self): if not self.result: raise Exception("No result yet.") return int(self.result["code"]) @property def headers(self): if not self.result: raise Exception("No result yet.") h = Headers() for i in self.result["headers"]: h.addRawHeader(*i) return h def writeHeaders(self, version, code, reason, headers): self.result["version"] = version self.result["code"] = code self.result["reason"] = reason self.result["headers"] = headers def write(self, content): assert isinstance(content, bytes), "Should be bytes! " + repr(content) if "body" not in self.result: self.result["body"] = b"" self.result["body"] += content def registerProducer(self, producer, streaming): self._producer = producer self.producerStreaming = streaming def _produce(): if self._producer: self._producer.resumeProducing() self._reactor.callLater(0.1, _produce) if not streaming: self._reactor.callLater(0.0, _produce) def unregisterProducer(self): if self._producer is None: return self._producer = None def requestDone(self, _self): self.result["done"] = True def getPeer(self): # We give an address so that getClientAddress().host returns a non null entry, # causing us to record the MAU return address.IPv4Address("TCP", "127.0.0.1", 3423) def getHost(self): return None @property def transport(self): return self def getPeerCertificate(self): """Returns the hardcoded TLS certificate for fake.server.""" return crypto.load_certificate(crypto.FILETYPE_PEM, FAKE_SERVER_CERT_PEM) class FakeSite: """A fake Twisted Web Site.""" pass def make_request( reactor, method, path, content=b"", access_token=None, request=Request, shorthand=True, federation_auth_origin=None, ): """ Make a web request using the given method and path, feed it the content, and return the Request and the Channel underneath. Mostly Args: reactor (IReactor): The Twisted reactor to use when performing the request. method (bytes or unicode): The HTTP request method ("verb"). path (bytes or unicode): The HTTP path, suitably URL encoded (e.g. escaped UTF-8 & spaces and such). content (bytes or dict): The body of the request. JSON-encoded, if a dict. access_token (unicode): An access token to use to authenticate the request, None if no access token needs to be included. request (IRequest): The class to use when instantiating the request object. shorthand: Whether to try and be helpful and prefix the given URL with the usual REST API path, if it doesn't contain it. federation_auth_origin (bytes|None): if set to not-None, we will add a fake Authorization header pretenting to be the given server name. Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ if not isinstance(method, bytes): method = method.encode("ascii") if not isinstance(path, bytes): path = path.encode("ascii") # Decorate it to be the full path, if we're using shorthand if shorthand and not path.startswith(b"/_matrix"): path = b"/_matrix/identity/v2/" + path path = path.replace(b"//", b"/") if not path.startswith(b"/"): path = b"/" + path if isinstance(content, dict): content = json.dumps(content) if isinstance(content, str): content = content.encode("utf8") site = FakeSite() channel = FakeChannel(site, reactor) req = request(channel) req.process = lambda: b"" req.content = BytesIO(content) req.postpath = list(map(unquote, path[1:].split(b"/"))) if access_token: req.requestHeaders.addRawHeader( b"Authorization", b"Bearer " + access_token.encode("ascii") ) if federation_auth_origin is not None: req.requestHeaders.addRawHeader( b"Authorization", b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin,), ) if content: req.requestHeaders.addRawHeader(b"Content-Type", b"application/json") req.requestReceived(method, path, b"1.1") return req, channel class ToTwistedHandler(logging.Handler): """logging handler which sends the logs to the twisted log""" tx_log = twisted.logger.Logger() def emit(self, record): log_entry = self.format(record) log_level = record.levelname.lower().replace("warning", "warn") self.tx_log.emit( twisted.logger.LogLevel.levelWithName(log_level), "{entry}", entry=log_entry ) def setup_logging(): """Configure the python logging appropriately for the tests. (Logs will end up in _trial_temp.) """ root_logger = logging.getLogger() log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s" handler = ToTwistedHandler() formatter = logging.Formatter(log_format) handler.setFormatter(formatter) root_logger.addHandler(handler) log_level = os.environ.get("SYDENT_TEST_LOG_LEVEL", "ERROR") root_logger.setLevel(log_level) setup_logging() @implementer(IReactorPluggableNameResolver) class ResolvingMemoryReactorClock(MemoryReactorClock): """ A MemoryReactorClock that supports name resolution. """ def __init__(self): lookups = self.lookups = {} # type: Dict[str, str] @implementer(IResolverSimple) class FakeResolver: def getHostByName(self, name, timeout=None): if name not in lookups: return fail(DNSLookupError("OH NO: unknown %s" % (name,))) return succeed(lookups[name]) self.nameResolver = SimpleResolverComplexifier(FakeResolver()) super().__init__() def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver: raise NotImplementedError() class AsyncMock(MagicMock): async def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs)