update comment

.ci/scripts/checkout_complement.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-#
-# Fetches a version of complement which best matches the current build.
-#
-# The tarball is unpacked into `./complement`.
-
-set -e
-mkdir -p complement
-
-# Pick an appropriate version of complement. Depending on whether this is a PR or release,
-# etc. we need to use different fallbacks:
-#
-# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-#    for pull requests, otherwise GITHUB_REF).
-# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-#    (GITHUB_BASE_REF for pull requests).
-# 3. Use the default complement branch ("HEAD").
-for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
-  # Skip empty branch names and merge commits.
-  if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-    continue
-  fi
-
-  (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-done

.git-blame-ignore-revs

@@ -6,6 +6,3 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56
 
 # Update black to 20.8b1 (#9381).
 0a00b7ff14890987f09112a2ae696c61001e6cf1
-
-# Convert tests/rest/admin/test_room.py to unix file endings (#7953).
-c4268e3da64f1abb5b31deaeb5769adb6510c0a7

.github/workflows/docker.yml

@@ -34,24 +34,32 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      # TODO: consider using https://github.com/docker/metadata-action instead of this
+      # custom magic
       - name: Calculate docker image tag
         id: set-tag
-        uses: docker/metadata-action@master
-        with:
-          images: matrixdotorg/synapse
-          flavor: |
-            latest=false
-          tags: |
-            type=raw,value=develop,enable=${{ github.ref == 'refs/heads/develop' }}
-            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
-            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=pep440,pattern={{raw}}
+        run: |
+          case "${GITHUB_REF}" in
+              refs/heads/develop)
+                  tag=develop
+                  ;;
+              refs/heads/master|refs/heads/main)
+                  tag=latest
+                  ;;
+              refs/tags/*)
+                  tag=${GITHUB_REF#refs/tags/}
+                  ;;
+              *)
+                  tag=${GITHUB_SHA}
+                  ;;
+          esac
+          echo "::set-output name=tag::$tag"
 
       - name: Build and push all platforms
         uses: docker/build-push-action@v2
         with:
           push: true
           labels: "gitsha1=${{ github.sha }}"
-          tags: "${{ steps.set-tag.outputs.tags }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
           file: "docker/Dockerfile"
           platforms: linux/amd64,linux/arm64

.github/workflows/latest_deps.yml

@@ -32,15 +32,12 @@ jobs:
         with:
           python-version: "3.x"
           poetry-version: "1.2.0b1"
-          extras: "all"
       # Dump installed versions for debugging.
       - run: poetry run pip list > before.txt
       # Upgrade all runtime dependencies only. This is intended to mimic a fresh
       # `pip install matrix-synapse[all]` as closely as possible.
       - run: poetry update --no-dev
       - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
-      - name: Remove warn_unused_ignores from mypy config
-        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
   trial:
     runs-on: ubuntu-latest

.github/workflows/tests.yml

@@ -306,7 +306,7 @@ jobs:
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
-    if: "${{ !failure() && !cancelled() }}"
+    if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
 
@@ -333,7 +333,26 @@ jobs:
       # Attempt to check out the same branch of Complement as the PR. If it
       # doesn't exist, fallback to HEAD.
       - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("HEAD").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
 
       - run: |
           set -o pipefail
@@ -341,45 +360,6 @@ jobs:
         shell: bash
         name: Run Complement Tests
 
-  # We only run the workers tests on `develop` for now, because they're too slow to wait for on PRs.
-  # Sadly, you can't have an `if` condition on the value of a matrix, so this is a temporary, separate job for now.
-  # GitHub Actions doesn't support YAML anchors, so it's full-on duplication for now.
-  complement-developonly:
-    if: "${{ !failure() && !cancelled() && (github.ref == 'refs/heads/develop') }}"
-    needs: linting-done
-    runs-on: ubuntu-latest
-
-    steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
-      - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
-
-      - run: |
-          set -o pipefail
-          WORKERS=1 COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
-
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}

.github/workflows/twisted_trunk.yml

@@ -24,8 +24,6 @@ jobs:
           poetry remove twisted
           poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry install --no-interaction --extras "all test"
-      - name: Remove warn_unused_ignores from mypy config
-        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
 
   trial:

CHANGES.md

@@ -1,300 +1,3 @@
-Synapse 1.60.0 (2022-05-31)
-===========================
-
-This release of Synapse adds a unique index to the `state_group_edges` table, in
-order to prevent accidentally introducing duplicate information (for example,
-because a database backup was restored multiple times). If your Synapse database
-already has duplicate rows in this table, this could fail with an error and
-require manual remediation.
-
-Additionally, the signature of the `check_event_for_spam` module callback has changed.
-The previous signature has been deprecated and remains working for now. Module authors
-should update their modules to use the new signature where possible.
-
-See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1600)
-for more details.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.60.0rc1 that would break some imports from `synapse.module_api`. ([\#12918](https://github.com/matrix-org/synapse/issues/12918))
-
-
-Synapse 1.60.0rc2 (2022-05-27)
-==============================
-
-Features
---------
-
-- Add an option allowing users to use their password to reauthenticate for privileged actions even though password login is disabled. ([\#12883](https://github.com/matrix-org/synapse/issues/12883))
-
-
-Bugfixes
---------
-
-- Explicitly close `ijson` coroutines once we are done with them, instead of leaving the garbage collector to close them. ([\#12875](https://github.com/matrix-org/synapse/issues/12875))
-
-
-Internal Changes
-----------------
-
-- Improve URL previews by not including the content of media tags in the generated description. ([\#12887](https://github.com/matrix-org/synapse/issues/12887))
-
-
-Synapse 1.60.0rc1 (2022-05-24)
-==============================
-
-Features
---------
-
-- Measure the time taken in spam-checking callbacks and expose those measurements as metrics. ([\#12513](https://github.com/matrix-org/synapse/issues/12513))
-- Add a `default_power_level_content_override` config option to set default room power levels per room preset. ([\#12618](https://github.com/matrix-org/synapse/issues/12618))
-- Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). ([\#12623](https://github.com/matrix-org/synapse/issues/12623))
-- Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. ([\#12672](https://github.com/matrix-org/synapse/issues/12672), [\#12809](https://github.com/matrix-org/synapse/issues/12809))
-- Synapse will now reload [cache config](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#caching) when it receives a [SIGHUP](https://en.wikipedia.org/wiki/SIGHUP) signal. ([\#12673](https://github.com/matrix-org/synapse/issues/12673))
-- Add a config options to allow for auto-tuning of caches. ([\#12701](https://github.com/matrix-org/synapse/issues/12701))
-- Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. ([\#12718](https://github.com/matrix-org/synapse/issues/12718))
-- Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. ([\#12744](https://github.com/matrix-org/synapse/issues/12744))
-- Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). ([\#12786](https://github.com/matrix-org/synapse/issues/12786), [\#12792](https://github.com/matrix-org/synapse/issues/12792))
-- Update to the `check_event_for_spam` module callback. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808))
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.7.0 that would prevent events from being sent to clients if there's a retention policy in the room when the support for retention policies is disabled. ([\#12611](https://github.com/matrix-org/synapse/issues/12611))
-- Fix a bug introduced in Synapse 1.57.0 where `/messages` would throw a 500 error when querying for a non-existent room. ([\#12683](https://github.com/matrix-org/synapse/issues/12683))
-- Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. ([\#12687](https://github.com/matrix-org/synapse/issues/12687))
-- Fix a long-standing bug where an empty room would be created when a user with an insufficient power level tried to upgrade a room. ([\#12696](https://github.com/matrix-org/synapse/issues/12696))
-- Fix a bug introduced in Synapse 1.30.0 where empty rooms could be automatically created if a monthly active users limit is set. ([\#12713](https://github.com/matrix-org/synapse/issues/12713))
-- Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper. ([\#12721](https://github.com/matrix-org/synapse/issues/12721))
-- Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. ([\#12747](https://github.com/matrix-org/synapse/issues/12747))
-- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762))
-- Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. ([\#12770](https://github.com/matrix-org/synapse/issues/12770))
-- Give a meaningful error message when a client tries to create a room with an invalid alias localpart. ([\#12779](https://github.com/matrix-org/synapse/issues/12779))
-- Fix a bug introduced in 1.43.0 where a file (`providers.json`) was never closed. Contributed by @arkamar. ([\#12794](https://github.com/matrix-org/synapse/issues/12794))
-- Fix a long-standing bug where finished log contexts would be re-started when failing to contact remote homeservers. ([\#12803](https://github.com/matrix-org/synapse/issues/12803))
-- Fix a bug, introduced in Synapse 1.21.0, that led to media thumbnails being unusable before the index has been added in the background. ([\#12823](https://github.com/matrix-org/synapse/issues/12823))
-
-
-Updates to the Docker image
----------------------------
-
-- Fix the docker file after a dependency update. ([\#12853](https://github.com/matrix-org/synapse/issues/12853))
-
-
-Improved Documentation
-----------------------
-
-- Fix a typo in the Media Admin API documentation. ([\#12715](https://github.com/matrix-org/synapse/issues/12715))
-- Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. ([\#12727](https://github.com/matrix-org/synapse/issues/12727))
-- Fix typo in server listener documentation. ([\#12742](https://github.com/matrix-org/synapse/issues/12742))
-- Link to the configuration manual from the welcome page of the documentation. ([\#12748](https://github.com/matrix-org/synapse/issues/12748))
-- Fix typo in `run_background_tasks_on` option name in configuration manual documentation. ([\#12749](https://github.com/matrix-org/synapse/issues/12749))
-- Add information regarding the `rc_invites` ratelimiting option to the configuration docs. ([\#12759](https://github.com/matrix-org/synapse/issues/12759))
-- Add documentation for cancellation of request processing. ([\#12761](https://github.com/matrix-org/synapse/issues/12761))
-- Recommend using docker to run tests against postgres. ([\#12765](https://github.com/matrix-org/synapse/issues/12765))
-- Add missing user directory endpoint from the generic worker documentation. Contributed by @olmari. ([\#12773](https://github.com/matrix-org/synapse/issues/12773))
-- Add additional info to documentation of config option `cache_autotuning`. ([\#12776](https://github.com/matrix-org/synapse/issues/12776))
-- Update configuration manual documentation to document size-related suffixes. ([\#12777](https://github.com/matrix-org/synapse/issues/12777))
-- Fix invalid YAML syntax in the example documentation for the `url_preview_accept_language` config option. ([\#12785](https://github.com/matrix-org/synapse/issues/12785))
-
-
-Deprecations and Removals
--------------------------
-
-- Require a body in POST requests to `/rooms/{roomId}/receipt/{receiptType}/{eventId}`, as required by the [Matrix specification](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidreceiptreceipttypeeventid). This breaks compatibility with Element Android 1.2.0 and earlier: users of those clients will be unable to send read receipts. ([\#12709](https://github.com/matrix-org/synapse/issues/12709))
-
-
-Internal Changes
-----------------
-
-- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10533](https://github.com/matrix-org/synapse/issues/10533))
-- Preparation for faster-room-join work: return subsets of room state which we already have, immediately. ([\#12498](https://github.com/matrix-org/synapse/issues/12498))
-- Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. ([\#12586](https://github.com/matrix-org/synapse/issues/12586), [\#12588](https://github.com/matrix-org/synapse/issues/12588), [\#12630](https://github.com/matrix-org/synapse/issues/12630), [\#12694](https://github.com/matrix-org/synapse/issues/12694), [\#12698](https://github.com/matrix-org/synapse/issues/12698), [\#12699](https://github.com/matrix-org/synapse/issues/12699), [\#12700](https://github.com/matrix-org/synapse/issues/12700), [\#12705](https://github.com/matrix-org/synapse/issues/12705))
-- Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. ([\#12708](https://github.com/matrix-org/synapse/issues/12708))
-- Improve documentation of the `synapse.push` module. ([\#12676](https://github.com/matrix-org/synapse/issues/12676))
-- Refactor functions to on `PushRuleEvaluatorForEvent`. ([\#12677](https://github.com/matrix-org/synapse/issues/12677))
-- Preparation for database schema simplifications: stop writing to `event_reference_hashes`. ([\#12679](https://github.com/matrix-org/synapse/issues/12679))
-- Remove code which updates unused database column `application_services_state.last_txn`. ([\#12680](https://github.com/matrix-org/synapse/issues/12680))
-- Refactor `EventContext` class. ([\#12689](https://github.com/matrix-org/synapse/issues/12689))
-- Remove an unneeded class in the push code. ([\#12691](https://github.com/matrix-org/synapse/issues/12691))
-- Consolidate parsing of relation information from events. ([\#12693](https://github.com/matrix-org/synapse/issues/12693))
-- Convert namespace class `Codes` into a string enum. ([\#12703](https://github.com/matrix-org/synapse/issues/12703))
-- Optimize private read receipt filtering. ([\#12711](https://github.com/matrix-org/synapse/issues/12711))
-- Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. ([\#12720](https://github.com/matrix-org/synapse/issues/12720))
-- Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. ([\#12723](https://github.com/matrix-org/synapse/issues/12723))
-- Update configs used by Complement to allow more invites/3PID validations during tests. ([\#12731](https://github.com/matrix-org/synapse/issues/12731))
-- Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. ([\#12769](https://github.com/matrix-org/synapse/issues/12769))
-- Move methods that call `add_push_rule` to the `PushRuleStore` class. ([\#12772](https://github.com/matrix-org/synapse/issues/12772))
-- Make handling of federation Authorization header (more) compliant with RFC7230. ([\#12774](https://github.com/matrix-org/synapse/issues/12774))
-- Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. ([\#12775](https://github.com/matrix-org/synapse/issues/12775))
-- Do not keep going if there are 5 back-to-back background update failures. ([\#12781](https://github.com/matrix-org/synapse/issues/12781))
-- Fix federation when using the demo scripts. ([\#12783](https://github.com/matrix-org/synapse/issues/12783))
-- The `hash_password` script now fails when it is called without specifying a config file. Contributed by @jae1911. ([\#12789](https://github.com/matrix-org/synapse/issues/12789))
-- Improve and fix type hints. ([\#12567](https://github.com/matrix-org/synapse/issues/12567), [\#12477](https://github.com/matrix-org/synapse/issues/12477), [\#12717](https://github.com/matrix-org/synapse/issues/12717), [\#12753](https://github.com/matrix-org/synapse/issues/12753), [\#12695](https://github.com/matrix-org/synapse/issues/12695), [\#12734](https://github.com/matrix-org/synapse/issues/12734), [\#12716](https://github.com/matrix-org/synapse/issues/12716), [\#12726](https://github.com/matrix-org/synapse/issues/12726), [\#12790](https://github.com/matrix-org/synapse/issues/12790), [\#12833](https://github.com/matrix-org/synapse/issues/12833))
-- Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. ([\#12791](https://github.com/matrix-org/synapse/issues/12791))
-- Remove Caddy from the Synapse workers image used in Complement. ([\#12818](https://github.com/matrix-org/synapse/issues/12818))
-- Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. ([\#12819](https://github.com/matrix-org/synapse/issues/12819))
-- Support registering Application Services when running with workers under Complement. ([\#12826](https://github.com/matrix-org/synapse/issues/12826))
-- Disable 'faster room join' Complement tests when testing against Synapse with workers. ([\#12842](https://github.com/matrix-org/synapse/issues/12842))
-
-
-Synapse 1.59.1 (2022-05-18)
-===========================
-
-This release fixes a long-standing issue which could prevent Synapse's user directory for updating properly.
-
-Bugfixes
-----------------
-
-- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. Contributed by Nick @ Beeper. ([\#12762](https://github.com/matrix-org/synapse/issues/12762))
-
-
-Synapse 1.59.0 (2022-05-17)
-===========================
-
-Synapse 1.59 makes several changes that server administrators should be aware of:
-
-- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616))
-- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654))
-
-See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details.
-
-Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597))
-
-
-Bugfixes
---------
-
-- Fix DB performance regression introduced in Synapse 1.59.0rc2. ([\#12745](https://github.com/matrix-org/synapse/issues/12745))
-
-
-Synapse 1.59.0rc2 (2022-05-16)
-==============================
-
-Note: this release candidate includes a performance regression which can cause database disruption. Other release candidates in the v1.59.0 series are not affected, and a fix will be included in the v1.59.0 final release.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was rejected. ([\#12729](https://github.com/matrix-org/synapse/issues/12729))
-
-
-Synapse 1.59.0rc1 (2022-05-10)
-==============================
-
-Features
---------
-
-- Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. ([\#11507](https://github.com/matrix-org/synapse/issues/11507))
-- Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670))
-- Extend the [module API](https://github.com/matrix-org/synapse/blob/release-v1.59/synapse/module_api/__init__.py) to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406))
-- Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452))
-- Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. ([\#12654](https://github.com/matrix-org/synapse/issues/12654))
-- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526))
-- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore `m.room.server_acl` events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601))
-- Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619))
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273))
-- Fix a bug introduced in Synapse 1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544))
-- Fix a bug introduced in Synapse 1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570))
-- Fix a long-standing bug where status codes would almost always get logged as `200!`, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580))
-- Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594))
-- Fix a bug introduced in Synapse 1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633))
-- Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657))
-- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656))
-
-
-Updates to the Docker image
----------------------------
-
-- Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. ([\#12541](https://github.com/matrix-org/synapse/issues/12541))
-- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by @henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573))
-
-
-Improved Documentation
-----------------------
-
-- Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536))
-- Add missing linebreak to `pipx` install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579))
-- Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621))
-- Fixes to the formatting of `README.rst`. ([\#12627](https://github.com/matrix-org/synapse/issues/12627))
-- Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). ([\#12596](https://github.com/matrix-org/synapse/issues/12596))
-- Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from
-  [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). ([\#12597](https://github.com/matrix-org/synapse/issues/12597))
-- Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. ([\#12613](https://github.com/matrix-org/synapse/issues/12613))
-
-
-Internal Changes
-----------------
-
-- Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480))
-- Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500))
-- Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505))
-- Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564))
-- Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568))
-- Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577))
-- Improve docstrings for the receipts store. ([\#12581](https://github.com/matrix-org/synapse/issues/12581))
-- Use constants for read-receipts in tests. ([\#12582](https://github.com/matrix-org/synapse/issues/12582))
-- Log status code of cancelled requests as 499 and avoid logging stack traces for them. ([\#12587](https://github.com/matrix-org/synapse/issues/12587), [\#12663](https://github.com/matrix-org/synapse/issues/12663))
-- Remove special-case for `twisted` logger from default log config. ([\#12589](https://github.com/matrix-org/synapse/issues/12589))
-- Use `getClientAddress` instead of the deprecated `getClientIP`. ([\#12599](https://github.com/matrix-org/synapse/issues/12599))
-- Add link to documentation in Grafana Dashboard. ([\#12602](https://github.com/matrix-org/synapse/issues/12602))
-- Reduce log spam when running multiple event persisters. ([\#12610](https://github.com/matrix-org/synapse/issues/12610))
-- Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614))
-- Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616))
-- Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620))
-- Remove use of the `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624))
-- Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632))
-- Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637))
-- Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652))
-- Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665))
-- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556))
-- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612))
-
-### Typechecking
-
-- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356))
-- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485))
-- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531))
-- Allow unused `# type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576))
-- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608))
-- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650))
-- Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666))
-- Use `ParamSpec` to refine type hints. ([\#12667](https://github.com/matrix-org/synapse/issues/12667))
-- Fix mypy against latest pillow stubs. ([\#12671](https://github.com/matrix-org/synapse/issues/12671))
-
-Synapse 1.58.1 (2022-05-05)
-===========================
-
-This patch release includes a fix to the Debian packages, installing the
-`systemd` and `cache_memory` extra package groups, which were incorrectly
-omitted in v1.58.0. This primarily prevented Synapse from starting
-when the `systemd.journal.JournalHandler` log handler was configured.
-See [#12631](https://github.com/matrix-org/synapse/issues/12631) for further information.
-
-Otherwise, no significant changes since 1.58.0.
-
-
-Synapse 1.58.0 (2022-05-03)
-===========================
-
-As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61.
-
-No significant changes since 1.58.0rc2.
-
-
 Synapse 1.58.0rc2 (2022-04-26)
 ==============================
 
@@ -316,6 +19,8 @@ Internal Changes
 Synapse 1.58.0rc1 (2022-04-26)
 ==============================
 
+As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61.
+
 Features
 --------
 

README.rst

@@ -55,7 +55,7 @@ solutions. The hope is for Matrix to act as the building blocks for a new
 generation of fully open and interoperable messaging and VoIP apps for the
 internet.
 
-Synapse is a Matrix "homeserver" implementation developed by the matrix.org core
+Synapse is a Matrix "homeserver" implementation developed by the matrix.org core 
 team, written in Python 3/Twisted.
 
 In Matrix, every user runs one or more Matrix clients, which connect through to
@@ -294,13 +294,13 @@ directory of your choice::
     cd synapse
 
 Synapse has a number of external dependencies. We maintain a fixed development
-environment using `Poetry <https://python-poetry.org/>`_. First, install poetry. We recommend::
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend
 
     pip install --user pipx
     pipx install poetry
 
 as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
-(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`_
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
 for other installation methods.) Then ask poetry to create a virtual environment
 from the project and install Synapse's dependencies::
 
@@ -309,11 +309,11 @@ from the project and install Synapse's dependencies::
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
 
-We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`::
+We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
     poetry run ./demo/start.sh
 
-(to stop, you can use ``poetry run ./demo/stop.sh``)
+(to stop, you can use `poetry run ./demo/stop.sh`)
 
 See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
 for more information.

changelog.d/12356.misc

@@ -0,0 +1 @@
+Fix scripts-dev to pass typechecking.

changelog.d/12406.feature

@@ -0,0 +1 @@
+Add a module API to allow modules to change actions for existing push rules of local users.

changelog.d/12480.misc

@@ -0,0 +1 @@
+Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time.

changelog.d/12485.misc

@@ -0,0 +1 @@
+Add some type hints to datastore.

changelog.d/12505.misc

@@ -0,0 +1 @@
+Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests.

changelog.d/12526.feature

@@ -0,0 +1 @@
+Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.

changelog.d/12531.misc

@@ -0,0 +1 @@
+Remove unused `# type: ignore`s.

changelog.d/12541.docker

@@ -0,0 +1 @@
+Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments.

changelog.d/12544.bugfix

@@ -0,0 +1 @@
+Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper.

changelog.d/12553.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12558.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12563.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12564.misc

@@ -0,0 +1 @@
+Consistently check if an object is a `frozendict`.

changelog.d/12572.feature

@@ -0,0 +1 @@
+Add the Synapse function `types.map_username_to_mxid_localpart` to the Module API.

changelog.d/12732.feature

@@ -1 +0,0 @@
-Add new `media_retention` options to the homeserver config for routinely cleaning up non-recently accessed media.

changelog.d/12740.feature

@@ -1 +0,0 @@
-Experimental support for [MSC3772](https://github.com/matrix-org/matrix-spec-proposals/pull/3772): Push rule for mutually related events.

changelog.d/12746.bugfix

@@ -1 +0,0 @@
-Always send an `access_token` in `/thirdparty/` requests to appservices, as required by the [Matrix specification](https://spec.matrix.org/v1.1/application-service-api/#third-party-networks).

changelog.d/12766.bugfix

@@ -1 +0,0 @@
-Implement [MSC3816](https://github.com/matrix-org/matrix-spec-proposals/pull/3816): sending the root event in a thread should count as "participated" in it.

changelog.d/12784.bugfix

@@ -1 +0,0 @@
-Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API.

changelog.d/12808.feature

@@ -1 +0,0 @@
-Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes).

changelog.d/12810.misc

@@ -1 +0,0 @@
-Test Synapse against Complement with workers.

changelog.d/12811.misc

@@ -1 +0,0 @@
-Reduce the amount of state we pull from the DB.

changelog.d/12812.misc

@@ -1 +0,0 @@
-Try other homeservers when re-syncing state for rooms with partial state.

changelog.d/12813.misc

@@ -1 +0,0 @@
-Resume state re-syncing for rooms with partial state after a Synapse restart.

changelog.d/12829.bugfix

@@ -1 +0,0 @@
-Fix a bug where we did not correctly handle invalid device list updates over federation. Contributed by Carl Bordum Hansen.

changelog.d/12832.bugfix

@@ -1 +0,0 @@
-Fixed a bug which allowed multiple async operations to access database locks concurrently. Contributed by @sumnerevans @ Beeper.

changelog.d/12836.misc

@@ -1 +0,0 @@
-Remove Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) endpoint dependency on the User Directory.

changelog.d/12838.feature

@@ -1 +0,0 @@
-Add storage and module API methods to get monthly active users (and their corresponding appservices) within an optionally specified time range.

changelog.d/12840.bugfix

@@ -1 +0,0 @@
-Fix an issue introduced in Synapse 0.34 where the `/notifications` endpoint would only return notifications if a user registered at least one pusher. Contributed by Famedly.

changelog.d/12843.bugfix

@@ -1 +0,0 @@
-Fix bug where servers using a Postgres database would fail to backfill from an insertion event when MSC2716 is enabled (`experimental_features.msc2716_enabled`).

changelog.d/12845.feature

@@ -1 +0,0 @@
-Support the new error code "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823).

changelog.d/12846.misc

@@ -1 +0,0 @@
-Experimental: expand `check_event_for_spam` with ability to return additional fields. This enables spam-checker implementations to experiment with mechanisms to give users more information about why they are blocked and whether any action is needed from them to be unblocked.

changelog.d/12849.misc

@@ -1 +0,0 @@
-Remove `dont_notify` from the `.m.rule.room.server_acl` rule.

changelog.d/12851.misc

@@ -1 +0,0 @@
-Remove the unstable `/hierarchy` endpoint from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/12852.misc

@@ -1 +0,0 @@
-Pull out less state when handling gaps in room DAG.

changelog.d/12855.feature

@@ -1 +0,0 @@
-Add a configurable background job to delete stale devices.

changelog.d/12856.misc

@@ -1 +0,0 @@
-Clean-up the push rules datastore.

changelog.d/12858.bugfix

@@ -1 +0,0 @@
-Fix [MSC3787](https://github.com/matrix-org/matrix-spec-proposals/pull/3787) rooms being omitted from room directory, room summary and space hierarchy responses.

changelog.d/12859.feature

@@ -1 +0,0 @@
-Experimental support for [MSC3772](https://github.com/matrix-org/matrix-spec-proposals/pull/3772): Push rule for mutually related events.

changelog.d/12860.misc

@@ -1 +0,0 @@
-Correct a type annotation in the URL preview source code.

changelog.d/12863.doc

@@ -1 +0,0 @@
-Fix typos in documentation.

changelog.d/12865.misc

@@ -1 +0,0 @@
-Update `pyjwt` dependency to [2.4.0](https://github.com/jpadilla/pyjwt/releases/tag/2.4.0).

changelog.d/12866.misc

@@ -1 +0,0 @@
-Enable the `/account/whoami` endpoint on synapse worker processes. Contributed by Nick @ Beeper.

changelog.d/12867.doc

@@ -1 +0,0 @@
-Fix documentation incorrectly stating the `sendToDevice` endpoint can be directed at generic workers. Contributed by Nick @ Beeper.

changelog.d/12868.misc

@@ -1 +0,0 @@
-Enable the `batch_send` endpoint on synapse worker processes. Contributed by Nick @ Beeper.

changelog.d/12869.misc

@@ -1 +0,0 @@
-Don't generate empty AS transactions when the AS is flagged as down. Contributed by Nick @ Beeper.

changelog.d/12871.misc

@@ -1 +0,0 @@
-Fix up the variable `state_store` naming.

changelog.d/12872.misc

@@ -1 +0,0 @@
-Faster room joins: when querying the current state of the room, wait for state to be populated.

changelog.d/12877.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.54 which could sometimes cause exceptions when handling federated traffic.

changelog.d/12879.misc

@@ -1 +0,0 @@
-Avoid running queries which will never result in deletions.

changelog.d/12884.misc

@@ -1 +0,0 @@
-Use constants for EDU types.

changelog.d/12885.misc

@@ -1 +0,0 @@
-Reduce database load of `/sync` when presence is enabled.

changelog.d/12886.misc

@@ -1 +0,0 @@
-Refactor `have_seen_events` to reduce memory consumed when processing federation traffic.

changelog.d/12888.misc

@@ -1 +0,0 @@
-Refactor receipt linearization code.

changelog.d/12889.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.59.0 which caused room deletion to fail with a foreign key violation.

changelog.d/12894.misc

@@ -1 +0,0 @@
-Add type annotations to `synapse.logging.opentracing`.

changelog.d/12895.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12897.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12899.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12900.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12902.misc

@@ -1 +0,0 @@
-Remove PyNaCl occurrences directly used in Synapse code.

changelog.d/12903.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug which caused the `/messages` endpoint to return an incorrect `end` attribute when there were no more events. Contributed by @Vetchu.

changelog.d/12904.misc

@@ -1 +0,0 @@
-Pull out less state when handling gaps in room DAG.

changelog.d/12905.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was a redaction of an event that has since been purged.

changelog.d/12908.removal

@@ -1 +0,0 @@
-Remove contributed `kick_users.py` script. This is broken under Python 3, and is not added to the environment when `pip install`ing Synapse.

changelog.d/12909.removal

@@ -1 +0,0 @@
-Remove `contrib/jitsimeetbridge`. This was an unused experiment that hasn't been meaningfully changed since 2014.

changelog.d/12910.removal

@@ -1 +0,0 @@
-Remove unused `contrib/experiements/cursesio.py` script, which fails to run under Python 3.

changelog.d/12911.removal

@@ -1 +0,0 @@
-Remove unused `contrib/experiements/test_messaging.py` script. This fails to run on Python 3.

changelog.d/12912.misc

@@ -1 +0,0 @@
-Bump types-jsonschema from 4.4.1 to 4.4.6.

changelog.d/12913.misc

@@ -1 +0,0 @@
-Rename storage classes.

changelog.d/12914.misc

@@ -1 +0,0 @@
-Preparation for database schema simplifications: stop reading from `event_edges.room_id`.

changelog.d/12916.misc

@@ -1 +0,0 @@
-Check if we are in a virtual environment before overriding the `PYTHONPATH` environment variable in the demo script.

changelog.d/12917.feature

@@ -1 +0,0 @@
-Add storage and module API methods to get monthly active users (and their corresponding appservices) within an optionally specified time range.

changelog.d/12923.feature

@@ -1 +0,0 @@
-Support the new error code "ORG.MATRIX.MSC3823.USER_ACCOUNT_SUSPENDED" from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823).

changelog.d/12925.misc

@@ -1 +0,0 @@
-Improve the logging when signature checks on events fail.

changelog.d/12932.bugfix

@@ -1 +0,0 @@
-Fix potential memory leak when generating thumbnails.

changelog.d/12933.misc

@@ -1 +0,0 @@
-Test Synapse against Complement with workers.

changelog.d/12936.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12950.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where a URL preview would break if the image failed to download.

changelog.d/12951.feature

@@ -1 +0,0 @@
-Improve URL previews for pages with empty elements.

changelog.d/12952.feature

@@ -1 +0,0 @@
-Allow updating a user's password using the admin API without logging out their devices. Contributed by @jcgruenhage.

changelog.d/12964.misc

@@ -1 +0,0 @@
-Reduce the amount of state we pull from the DB.

changelog.d/12966.removal

@@ -1 +0,0 @@
-Remove support for the non-standard groups/communities feature from Synapse.

changelog.d/12974.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in v1.60.0 that causes synapse to fail to start when python module sqlite3 is not installed.

contrib/cmdclient/console.py

@@ -16,7 +16,6 @@
 
 """ Starts a synapse client console. """
 import argparse
-import binascii
 import cmd
 import getpass
 import json
@@ -27,8 +26,9 @@ import urllib
 from http import TwistedHttpClient
 from typing import Optional
 
+import nacl.encoding
+import nacl.signing
 import urlparse
-from signedjson.key import NACL_ED25519, decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 
 from twisted.internet import defer, reactor, threads
@@ -41,6 +41,7 @@ TRUSTED_ID_SERVERS = ["localhost:8001"]
 
 
 class SynapseCmd(cmd.Cmd):
+
     """Basic synapse command-line processor.
 
     This processes commands from the user and calls the relevant HTTP methods.
@@ -419,8 +420,8 @@ class SynapseCmd(cmd.Cmd):
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
                 if "public_key" in pubKeyObj:
-                    pubKey = decode_verify_key_bytes(
-                        NACL_ED25519, binascii.unhexlify(pubKeyObj["public_key"])
+                    pubKey = nacl.signing.VerifyKey(
+                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
                     )
                 else:
                     print("No public key found in pubkey response!")

contrib/experiments/cursesio.py

@@ -0,0 +1,165 @@
+# Copyright 2014-2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import curses
+import curses.wrapper
+from curses.ascii import isprint
+
+from twisted.internet import reactor
+
+
+class CursesStdIO:
+    def __init__(self, stdscr, callback=None):
+        self.statusText = "Synapse test app -"
+        self.searchText = ""
+        self.stdscr = stdscr
+
+        self.logLine = ""
+
+        self.callback = callback
+
+        self._setup()
+
+    def _setup(self):
+        self.stdscr.nodelay(1)  # Make non blocking
+
+        self.rows, self.cols = self.stdscr.getmaxyx()
+        self.lines = []
+
+        curses.use_default_colors()
+
+        self.paintStatus(self.statusText)
+        self.stdscr.refresh()
+
+    def set_callback(self, callback):
+        self.callback = callback
+
+    def fileno(self):
+        """We want to select on FD 0"""
+        return 0
+
+    def connectionLost(self, reason):
+        self.close()
+
+    def print_line(self, text):
+        """add a line to the internal list of lines"""
+
+        self.lines.append(text)
+        self.redraw()
+
+    def print_log(self, text):
+        self.logLine = text
+        self.redraw()
+
+    def redraw(self):
+        """method for redisplaying lines based on internal list of lines"""
+
+        self.stdscr.clear()
+        self.paintStatus(self.statusText)
+        i = 0
+        index = len(self.lines) - 1
+        while i < (self.rows - 3) and index >= 0:
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
+            i = i + 1
+            index = index - 1
+
+        self.printLogLine(self.logLine)
+
+        self.stdscr.refresh()
+
+    def paintStatus(self, text):
+        if len(text) > self.cols:
+            raise RuntimeError("TextTooLongError")
+
+        self.stdscr.addstr(
+            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
+
+    def printLogLine(self, text):
+        self.stdscr.addstr(
+            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
+
+    def doRead(self):
+        """Input is ready!"""
+        curses.noecho()
+        c = self.stdscr.getch()  # read a character
+
+        if c == curses.KEY_BACKSPACE:
+            self.searchText = self.searchText[:-1]
+
+        elif c == curses.KEY_ENTER or c == 10:
+            text = self.searchText
+            self.searchText = ""
+
+            self.print_line(">> %s" % text)
+
+            try:
+                if self.callback:
+                    self.callback.on_line(text)
+            except Exception as e:
+                self.print_line(str(e))
+
+            self.stdscr.refresh()
+
+        elif isprint(c):
+            if len(self.searchText) == self.cols - 2:
+                return
+            self.searchText = self.searchText + chr(c)
+
+        self.stdscr.addstr(
+            self.rows - 1,
+            0,
+            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
+        )
+
+        self.paintStatus(self.statusText + " %d" % len(self.searchText))
+        self.stdscr.move(self.rows - 1, len(self.searchText))
+        self.stdscr.refresh()
+
+    def logPrefix(self):
+        return "CursesStdIO"
+
+    def close(self):
+        """clean up"""
+
+        curses.nocbreak()
+        self.stdscr.keypad(0)
+        curses.echo()
+        curses.endwin()
+
+
+class Callback:
+    def __init__(self, stdio):
+        self.stdio = stdio
+
+    def on_line(self, text):
+        self.stdio.print_line(text)
+
+
+def main(stdscr):
+    screen = CursesStdIO(stdscr)  # create Screen object
+
+    callback = Callback(screen)
+
+    screen.set_callback(callback)
+
+    stdscr.refresh()
+    reactor.addReader(screen)
+    reactor.run()
+    screen.close()
+
+
+if __name__ == "__main__":
+    curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -0,0 +1,367 @@
+# Copyright 2014-2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+""" This is an example of using the server to server implementation to do a
+basic chat style thing. It accepts commands from stdin and outputs to stdout.
+
+It assumes that ucids are of the form <user>@<domain>, and uses <domain> as
+the address of the remote home server to hit.
+
+Usage:
+    python test_messaging.py <port>
+
+Currently assumes the local address is localhost:<port>
+
+"""
+
+
+import argparse
+import curses.wrapper
+import json
+import logging
+import os
+import re
+
+import cursesio
+
+from twisted.internet import defer, reactor
+from twisted.python import log
+
+from synapse.app.homeserver import SynapseHomeServer
+from synapse.federation import ReplicationHandler
+from synapse.federation.units import Pdu
+from synapse.util import origin_from_ucid
+
+# from synapse.logging.utils import log_function
+
+
+logger = logging.getLogger("example")
+
+
+def excpetion_errback(failure):
+    logging.exception(failure)
+
+
+class InputOutput:
+    """This is responsible for basic I/O so that a user can interact with
+    the example app.
+    """
+
+    def __init__(self, screen, user):
+        self.screen = screen
+        self.user = user
+
+    def set_home_server(self, server):
+        self.server = server
+
+    def on_line(self, line):
+        """This is where we process commands."""
+
+        try:
+            m = re.match(r"^join (\S+)$", line)
+            if m:
+                # The `sender` wants to join a room.
+                (room_name,) = m.groups()
+                self.print_line("%s joining %s" % (self.user, room_name))
+                self.server.join_room(room_name, self.user, self.user)
+                # self.print_line("OK.")
+                return
+
+            m = re.match(r"^invite (\S+) (\S+)$", line)
+            if m:
+                # `sender` wants to invite someone to a room
+                room_name, invitee = m.groups()
+                self.print_line("%s invited to %s" % (invitee, room_name))
+                self.server.invite_to_room(room_name, self.user, invitee)
+                # self.print_line("OK.")
+                return
+
+            m = re.match(r"^send (\S+) (.*)$", line)
+            if m:
+                # `sender` wants to message a room
+                room_name, body = m.groups()
+                self.print_line("%s send to %s" % (self.user, room_name))
+                self.server.send_message(room_name, self.user, body)
+                # self.print_line("OK.")
+                return
+
+            m = re.match(r"^backfill (\S+)$", line)
+            if m:
+                # we want to backfill a room
+                (room_name,) = m.groups()
+                self.print_line("backfill %s" % room_name)
+                self.server.backfill(room_name)
+                return
+
+            self.print_line("Unrecognized command")
+
+        except Exception as e:
+            logger.exception(e)
+
+    def print_line(self, text):
+        self.screen.print_line(text)
+
+    def print_log(self, text):
+        self.screen.print_log(text)
+
+
+class IOLoggerHandler(logging.Handler):
+    def __init__(self, io):
+        logging.Handler.__init__(self)
+        self.io = io
+
+    def emit(self, record):
+        if record.levelno < logging.WARN:
+            return
+
+        msg = self.format(record)
+        self.io.print_log(msg)
+
+
+class Room:
+    """Used to store (in memory) the current membership state of a room, and
+    which home servers we should send PDUs associated with the room to.
+    """
+
+    def __init__(self, room_name):
+        self.room_name = room_name
+        self.invited = set()
+        self.participants = set()
+        self.servers = set()
+
+        self.oldest_server = None
+
+        self.have_got_metadata = False
+
+    def add_participant(self, participant):
+        """Someone has joined the room"""
+        self.participants.add(participant)
+        self.invited.discard(participant)
+
+        server = origin_from_ucid(participant)
+        self.servers.add(server)
+
+        if not self.oldest_server:
+            self.oldest_server = server
+
+    def add_invited(self, invitee):
+        """Someone has been invited to the room"""
+        self.invited.add(invitee)
+        self.servers.add(origin_from_ucid(invitee))
+
+
+class HomeServer(ReplicationHandler):
+    """A very basic home server implentation that allows people to join a
+    room and then invite other people.
+    """
+
+    def __init__(self, server_name, replication_layer, output):
+        self.server_name = server_name
+        self.replication_layer = replication_layer
+        self.replication_layer.set_handler(self)
+
+        self.joined_rooms = {}
+
+        self.output = output
+
+    def on_receive_pdu(self, pdu):
+        """We just received a PDU"""
+        pdu_type = pdu.pdu_type
+
+        if pdu_type == "sy.room.message":
+            self._on_message(pdu)
+        elif pdu_type == "sy.room.member" and "membership" in pdu.content:
+            if pdu.content["membership"] == "join":
+                self._on_join(pdu.context, pdu.state_key)
+            elif pdu.content["membership"] == "invite":
+                self._on_invite(pdu.origin, pdu.context, pdu.state_key)
+        else:
+            self.output.print_line(
+                "#%s (unrec) %s = %s"
+                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            )
+
+    def _on_message(self, pdu):
+        """We received a message"""
+        self.output.print_line(
+            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
+        )
+
+    def _on_join(self, context, joinee):
+        """Someone has joined a room, either a remote user or a local user"""
+        room = self._get_or_create_room(context)
+        room.add_participant(joinee)
+
+        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
+
+    def _on_invite(self, origin, context, invitee):
+        """Someone has been invited"""
+        room = self._get_or_create_room(context)
+        room.add_invited(invitee)
+
+        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
+
+        if not room.have_got_metadata and origin is not self.server_name:
+            logger.debug("Get room state")
+            self.replication_layer.get_state_for_context(origin, context)
+            room.have_got_metadata = True
+
+    @defer.inlineCallbacks
+    def send_message(self, room_name, sender, body):
+        """Send a message to a room!"""
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            yield self.replication_layer.send_pdu(
+                Pdu.create_new(
+                    context=room_name,
+                    pdu_type="sy.room.message",
+                    content={"sender": sender, "body": body},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    @defer.inlineCallbacks
+    def join_room(self, room_name, sender, joinee):
+        """Join a room!"""
+        self._on_join(room_name, joinee)
+
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            pdu = Pdu.create_new(
+                context=room_name,
+                pdu_type="sy.room.member",
+                is_state=True,
+                state_key=joinee,
+                content={"membership": "join"},
+                origin=self.server_name,
+                destinations=destinations,
+            )
+            yield self.replication_layer.send_pdu(pdu)
+        except Exception as e:
+            logger.exception(e)
+
+    @defer.inlineCallbacks
+    def invite_to_room(self, room_name, sender, invitee):
+        """Invite someone to a room!"""
+        self._on_invite(self.server_name, room_name, invitee)
+
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            yield self.replication_layer.send_pdu(
+                Pdu.create_new(
+                    context=room_name,
+                    is_state=True,
+                    pdu_type="sy.room.member",
+                    state_key=invitee,
+                    content={"membership": "invite"},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    def backfill(self, room_name, limit=5):
+        room = self.joined_rooms.get(room_name)
+
+        if not room:
+            return
+
+        dest = room.oldest_server
+
+        return self.replication_layer.backfill(dest, room_name, limit)
+
+    def _get_room_remote_servers(self, room_name):
+        return list(self.joined_rooms.setdefault(room_name).servers)
+
+    def _get_or_create_room(self, room_name):
+        return self.joined_rooms.setdefault(room_name, Room(room_name))
+
+    def get_servers_for_context(self, context):
+        return defer.succeed(
+            self.joined_rooms.setdefault(context, Room(context)).servers
+        )
+
+
+def main(stdscr):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("user", type=str)
+    parser.add_argument("-v", "--verbose", action="count")
+    args = parser.parse_args()
+
+    user = args.user
+    server_name = origin_from_ucid(user)
+
+    # Set up logging
+
+    root_logger = logging.getLogger()
+
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
+    )
+    if not os.path.exists("logs"):
+        os.makedirs("logs")
+    fh = logging.FileHandler("logs/%s" % user)
+    fh.setFormatter(formatter)
+
+    root_logger.addHandler(fh)
+    root_logger.setLevel(logging.DEBUG)
+
+    # Hack: The only way to get it to stop logging to sys.stderr :(
+    log.theLogPublisher.observers = []
+    observer = log.PythonLoggingObserver()
+    observer.start()
+
+    # Set up synapse server
+
+    curses_stdio = cursesio.CursesStdIO(stdscr)
+    input_output = InputOutput(curses_stdio, user)
+
+    curses_stdio.set_callback(input_output)
+
+    app_hs = SynapseHomeServer(server_name, db_name="dbs/%s" % user)
+    replication = app_hs.get_replication_layer()
+
+    hs = HomeServer(server_name, replication, curses_stdio)
+
+    input_output.set_home_server(hs)
+
+    # Add input_output logger
+    io_logger = IOLoggerHandler(input_output)
+    io_logger.setFormatter(formatter)
+    root_logger.addHandler(io_logger)
+
+    # Start!
+
+    try:
+        port = int(server_name.split(":")[1])
+    except Exception:
+        port = 12345
+
+    app_hs.get_http_server().start_listening(port)
+
+    reactor.addReader(curses_stdio)
+
+    reactor.run()
+
+
+if __name__ == "__main__":
+    curses.wrapper(main)

contrib/grafana/synapse.json

@@ -66,18 +66,6 @@
       ],
       "title": "Dashboards",
       "type": "dashboards"
-    },
-    {
-      "asDropdown": false,
-      "icon": "external link",
-      "includeVars": false,
-      "keepTime": false,
-      "tags": [],
-      "targetBlank": true,
-      "title": "Synapse Documentation",
-      "tooltip": "Open Documentation",
-      "type": "link",
-      "url": "https://matrix-org.github.io/synapse/latest/"
     }
   ],
   "panels": [
@@ -10901,4 +10889,4 @@
   "title": "Synapse",
   "uid": "000000012",
   "version": 100
-}
+}

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python
+
+"""
+This is an attempt at bridging matrix clients into a Jitis meet room via Matrix
+video call.  It uses hard-coded xml strings overg XMPP BOSH. It can display one
+of the streams from the Jitsi bridge until the second lot of SDP comes down and
+we set the remote SDP at which point the stream ends. Our video never gets to
+the bridge.
+
+Requires:
+npm install jquery jsdom
+"""
+import json
+import subprocess
+import time
+
+import gevent
+import grequests
+from BeautifulSoup import BeautifulSoup
+
+ACCESS_TOKEN = ""
+
+MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
+MYUSERNAME = "@davetest:matrix.org"
+
+HTTPBIND = "https://meet.jit.si/http-bind"
+# HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+# ROOMNAME = "matrix"
+ROOMNAME = "pibble"
+
+HOST = "guest.jit.si"
+# HOST="jitsi.vuc.me"
+
+TURNSERVER = "turn.guest.jit.si"
+# TURNSERVER="turn.jitsi.vuc.me"
+
+ROOMDOMAIN = "meet.jit.si"
+# ROOMDOMAIN="conference.jitsi.vuc.me"
+
+
+class TrivialMatrixClient:
+    def __init__(self, access_token):
+        self.token = None
+        self.access_token = access_token
+
+    def getEvent(self):
+        while True:
+            url = (
+                MATRIXBASE
+                + "events?access_token="
+                + self.access_token
+                + "&timeout=60000"
+            )
+            if self.token:
+                url += "&from=" + self.token
+            req = grequests.get(url)
+            resps = grequests.map([req])
+            obj = json.loads(resps[0].content)
+            print("incoming from matrix", obj)
+            if "end" not in obj:
+                continue
+            self.token = obj["end"]
+            if len(obj["chunk"]):
+                return obj["chunk"][0]
+
+    def joinRoom(self, roomId):
+        url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
+        print(url)
+        headers = {"Content-Type": "application/json"}
+        req = grequests.post(url, headers=headers, data="{}")
+        resps = grequests.map([req])
+        obj = json.loads(resps[0].content)
+        print("response: ", obj)
+
+    def sendEvent(self, roomId, evType, event):
+        url = (
+            MATRIXBASE
+            + "rooms/"
+            + roomId
+            + "/send/"
+            + evType
+            + "?access_token="
+            + self.access_token
+        )
+        print(url)
+        print(json.dumps(event))
+        headers = {"Content-Type": "application/json"}
+        req = grequests.post(url, headers=headers, data=json.dumps(event))
+        resps = grequests.map([req])
+        obj = json.loads(resps[0].content)
+        print("response: ", obj)
+
+
+xmppClients = {}
+
+
+def matrixLoop():
+    while True:
+        ev = matrixCli.getEvent()
+        print(ev)
+        if ev["type"] == "m.room.member":
+            print("membership event")
+            if ev["membership"] == "invite" and ev["state_key"] == MYUSERNAME:
+                roomId = ev["room_id"]
+                print("joining room %s" % (roomId))
+                matrixCli.joinRoom(roomId)
+        elif ev["type"] == "m.room.message":
+            if ev["room_id"] in xmppClients:
+                print("already have a bridge for that user, ignoring")
+                continue
+            print("got message, connecting")
+            xmppClients[ev["room_id"]] = TrivialXmppClient(ev["room_id"], ev["user_id"])
+            gevent.spawn(xmppClients[ev["room_id"]].xmppLoop)
+        elif ev["type"] == "m.call.invite":
+            print("Incoming call")
+            # sdp = ev['content']['offer']['sdp']
+            # print "sdp: %s" % (sdp)
+            # xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            # gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev["type"] == "m.call.answer":
+            print("Call answered")
+            sdp = ev["content"]["answer"]["sdp"]
+            if ev["room_id"] not in xmppClients:
+                print("We didn't have a call for that room")
+                continue
+            # should probably check call ID too
+            xmppCli = xmppClients[ev["room_id"]]
+            xmppCli.sendAnswer(sdp)
+        elif ev["type"] == "m.call.hangup":
+            if ev["room_id"] in xmppClients:
+                xmppClients[ev["room_id"]].stop()
+                del xmppClients[ev["room_id"]]
+
+
+class TrivialXmppClient:
+    def __init__(self, matrixRoom, userId):
+        self.rid = 0
+        self.matrixRoom = matrixRoom
+        self.userId = userId
+        self.running = True
+
+    def stop(self):
+        self.running = False
+
+    def nextRid(self):
+        self.rid += 1
+        return "%d" % (self.rid)
+
+    def sendIq(self, xml):
+        fullXml = (
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>"
+            % (self.nextRid(), self.sid, xml)
+        )
+        # print "\t>>>%s" % (fullXml)
+        return self.xmppPoke(fullXml)
+
+    def xmppPoke(self, xml):
+        headers = {"Content-Type": "application/xml"}
+        req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
+        resps = grequests.map([req])
+        obj = BeautifulSoup(resps[0].content)
+        return obj
+
+    def sendAnswer(self, answer):
+        print("sdp from matrix client", answer)
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--sdp"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        jingle, out_err = p.communicate(answer)
+        jingle = jingle % {
+            "tojid": self.callfrom,
+            "action": "session-accept",
+            "initiator": self.callfrom,
+            "responder": self.jid,
+            "sid": self.callsid,
+        }
+        print("answer jingle from sdp", jingle)
+        res = self.sendIq(jingle)
+        print("reply from answer: ", res)
+
+        self.ssrcs = {}
+        jingleSoup = BeautifulSoup(jingle)
+        for cont in jingleSoup.iq.jingle.findAll("content"):
+            if cont.description:
+                self.ssrcs[cont["name"]] = cont.description["ssrc"]
+        print("my ssrcs:", self.ssrcs)
+
+        gevent.joinall([gevent.spawn(self.advertiseSsrcs)])
+
+    def advertiseSsrcs(self):
+        time.sleep(7)
+        print("SSRC spammer started")
+        while self.running:
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
+            res = self.sendIq(ssrcMsg)
+            print("reply from ssrc announce: ", res)
+            time.sleep(10)
+
+    def xmppLoop(self):
+        self.matrixCallId = time.time()
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), HOST)
+        )
+
+        print(res)
+        self.sid = res.body["sid"]
+        print("sid %s" % (self.sid))
+
+        res = self.sendIq(
+            "<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>"
+        )
+
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), self.sid, HOST)
+        )
+
+        res = self.sendIq(
+            "<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>"
+        )
+        print(res)
+
+        self.jid = res.body.iq.bind.jid.string
+        print("jid: %s" % (self.jid))
+        self.shortJid = self.jid.split("-")[0]
+
+        res = self.sendIq(
+            "<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>"
+        )
+
+        # randomthing = res.body.iq['to']
+        # whatsitpart = randomthing.split('-')[0]
+
+        # print "other random bind thing: %s" % (randomthing)
+
+        # advertise preence to the jitsi room, with our nick
+        res = self.sendIq(
+            "<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>"
+            % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)
+        )
+        self.muc = {"users": []}
+        for p in res.body.findAll("presence"):
+            u = {}
+            u["shortJid"] = p["from"].split("/")[1]
+            if p.c and p.c.nick:
+                u["nick"] = p.c.nick.string
+            self.muc["users"].append(u)
+        print("muc: ", self.muc)
+
+        # wait for stuff
+        while True:
+            print("waiting...")
+            res = self.sendIq("")
+            print("got from stream: ", res)
+            if res.body.iq:
+                jingles = res.body.iq.findAll("jingle")
+                if len(jingles):
+                    self.callfrom = res.body.iq["from"]
+                    self.handleInvite(jingles[0])
+            elif "type" in res.body and res.body["type"] == "terminate":
+                self.running = False
+                del xmppClients[self.matrixRoom]
+                return
+
+    def handleInvite(self, jingle):
+        self.initiator = jingle["initiator"]
+        self.callsid = jingle["sid"]
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--jingle"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        print("raw jingle invite", str(jingle))
+        sdp, out_err = p.communicate(str(jingle))
+        print("transformed remote offer sdp", sdp)
+        inviteEvent = {
+            "offer": {"type": "offer", "sdp": sdp},
+            "call_id": self.matrixCallId,
+            "version": 0,
+            "lifetime": 30000,
+        }
+        matrixCli.sendEvent(self.matrixRoom, "m.call.invite", inviteEvent)
+
+
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
+
+gevent.joinall([gevent.spawn(matrixLoop)])

contrib/jitsimeetbridge/syweb-jitsi-conference.patch

@@ -0,0 +1,188 @@
+diff --git a/syweb/webclient/app/components/matrix/matrix-call.js b/syweb/webclient/app/components/matrix/matrix-call.js
+index 9fbfff0..dc68077 100644
+--- a/syweb/webclient/app/components/matrix/matrix-call.js
++++ b/syweb/webclient/app/components/matrix/matrix-call.js
+@@ -16,6 +16,45 @@ limitations under the License.
+ 
+ 'use strict';
+ 
++
++function sendKeyframe(pc) {
++    console.log('sendkeyframe', pc.iceConnectionState);
++    if (pc.iceConnectionState !== 'connected') return; // safe...
++    pc.setRemoteDescription(
++        pc.remoteDescription,
++        function () {
++            pc.createAnswer(
++                function (modifiedAnswer) {
++                    pc.setLocalDescription(
++                        modifiedAnswer,
++                        function () {
++                            // noop
++                        },
++                        function (error) {
++                            console.log('triggerKeyframe setLocalDescription failed', error);
++                            messageHandler.showError();
++                        }
++                    );
++                },
++                function (error) {
++                    console.log('triggerKeyframe createAnswer failed', error);
++                    messageHandler.showError();
++                }
++            );
++        },
++        function (error) {
++            console.log('triggerKeyframe setRemoteDescription failed', error);
++            messageHandler.showError();
++        }
++    );
++} 
++
++
++
++
++
++
++
+ var forAllVideoTracksOnStream = function(s, f) {
+     var tracks = s.getVideoTracks();
+     for (var i = 0; i < tracks.length; i++) {
+@@ -83,7 +122,7 @@ angular.module('MatrixCall', [])
+     }
+ 
+     // FIXME: we should prevent any calls from being placed or accepted before this has finished
+-    MatrixCall.getTurnServer();
++    //MatrixCall.getTurnServer();
+ 
+     MatrixCall.CALL_TIMEOUT = 60000;
+     MatrixCall.FALLBACK_STUN_SERVER = 'stun:stun.l.google.com:19302';
+@@ -132,6 +171,22 @@ angular.module('MatrixCall', [])
+         pc.onsignalingstatechange = function() { self.onSignallingStateChanged(); };
+         pc.onicecandidate = function(c) { self.gotLocalIceCandidate(c); };
+         pc.onaddstream = function(s) { self.onAddStream(s); };
++
++        var datachan = pc.createDataChannel('RTCDataChannel', {
++            reliable: false
++        });
++        console.log("data chan: "+datachan);
++        datachan.onopen = function() {
++            console.log("data channel open");
++        };
++        datachan.onmessage = function() {
++            console.log("data channel message");
++        };
++        pc.ondatachannel = function(event) {
++            console.log("have data channel");
++            event.channel.binaryType = 'blob';
++        };
++
+         return pc;
+     }
+ 
+@@ -200,6 +255,12 @@ angular.module('MatrixCall', [])
+         }, this.msg.lifetime - event.age);
+     };
+ 
++    MatrixCall.prototype.receivedInvite = function(event) {
++        console.log("Got second invite for call "+this.call_id);
++        this.peerConn.setRemoteDescription(new RTCSessionDescription(this.msg.offer), this.onSetRemoteDescriptionSuccess, this.onSetRemoteDescriptionError);
++    };
++    
++
+     // perverse as it may seem, sometimes we want to instantiate a call with a hangup message
+     // (because when getting the state of the room on load, events come in reverse order and
+     // we want to remember that a call has been hung up)
+@@ -349,7 +410,7 @@ angular.module('MatrixCall', [])
+             'mandatory': {
+                 'OfferToReceiveAudio': true,
+                 'OfferToReceiveVideo': this.type == 'video'
+-            },
++            }
+         };
+         this.peerConn.createAnswer(function(d) { self.createdAnswer(d); }, function(e) {}, constraints);
+         // This can't be in an apply() because it's called by a predecessor call under glare conditions :(
+@@ -359,8 +420,20 @@ angular.module('MatrixCall', [])
+     MatrixCall.prototype.gotLocalIceCandidate = function(event) {
+         if (event.candidate) {
+             console.log("Got local ICE "+event.candidate.sdpMid+" candidate: "+event.candidate.candidate);
+-            this.sendCandidate(event.candidate);
+-        }
++            //this.sendCandidate(event.candidate);
++        } else {
++            console.log("have all candidates, sending answer");
++            var content = {
++                version: 0,
++                call_id: this.call_id,
++                answer: this.peerConn.localDescription
++            };
++            this.sendEventWithRetry('m.call.answer', content);
++            var self = this;
++            $rootScope.$apply(function() {
++                self.state = 'connecting';
++            });
++	}
+     }
+ 
+     MatrixCall.prototype.gotRemoteIceCandidate = function(cand) {
+@@ -418,15 +491,6 @@ angular.module('MatrixCall', [])
+         console.log("Created answer: "+description);
+         var self = this;
+         this.peerConn.setLocalDescription(description, function() {
+-            var content = {
+-                version: 0,
+-                call_id: self.call_id,
+-                answer: self.peerConn.localDescription
+-            };
+-            self.sendEventWithRetry('m.call.answer', content);
+-            $rootScope.$apply(function() {
+-                self.state = 'connecting';
+-            });
+         }, function() { console.log("Error setting local description!"); } );
+     };
+ 
+@@ -448,6 +512,9 @@ angular.module('MatrixCall', [])
+             $rootScope.$apply(function() {
+                 self.state = 'connected';
+                 self.didConnect = true;
++		/*$timeout(function() {
++                    sendKeyframe(self.peerConn);
++                }, 1000);*/
+             });
+         } else if (this.peerConn.iceConnectionState == 'failed') {
+             this.hangup('ice_failed');
+@@ -518,6 +585,7 @@ angular.module('MatrixCall', [])
+ 
+     MatrixCall.prototype.onRemoteStreamEnded = function(event) {
+         console.log("Remote stream ended");
++        return;
+         var self = this;
+         $rootScope.$apply(function() {
+             self.state = 'ended';
+diff --git a/syweb/webclient/app/components/matrix/matrix-phone-service.js b/syweb/webclient/app/components/matrix/matrix-phone-service.js
+index 55dbbf5..272fa27 100644
+--- a/syweb/webclient/app/components/matrix/matrix-phone-service.js
++++ b/syweb/webclient/app/components/matrix/matrix-phone-service.js
+@@ -48,6 +48,13 @@ angular.module('matrixPhoneService', [])
+                 return;
+             }
+ 
++            // do we already have an entry for this call ID?
++            var existingEntry = matrixPhoneService.allCalls[msg.call_id];
++            if (existingEntry) {
++                 existingEntry.receivedInvite(msg);
++                 return;
++            }
++
+             var call = undefined;
+             if (!isLive) {
+                 // if this event wasn't live then this call may already be over
+@@ -108,7 +115,7 @@ angular.module('matrixPhoneService', [])
+                     call.hangup();
+                 }
+             } else {
+-                $rootScope.$broadcast(matrixPhoneService.INCOMING_CALL_EVENT, call);
++                 $rootScope.$broadcast(matrixPhoneService.INCOMING_CALL_EVENT, call);
+             }
+         } else if (event.type == 'm.call.answer') {
+             var call = matrixPhoneService.allCalls[msg.call_id];

contrib/jitsimeetbridge/unjingle/strophe.jingle.sdp.js

@@ -0,0 +1,712 @@
+/* jshint -W117 */
+// SDP STUFF
+function SDP(sdp) {
+    this.media = sdp.split('\r\nm=');
+    for (var i = 1; i < this.media.length; i++) {
+        this.media[i] = 'm=' + this.media[i];
+        if (i != this.media.length - 1) {
+            this.media[i] += '\r\n';
+        }
+    }
+    this.session = this.media.shift() + '\r\n';
+    this.raw = this.session + this.media.join('');
+}
+
+exports.SDP = SDP;
+
+var jsdom          = require("jsdom");
+var window = jsdom.jsdom().parentWindow;
+var $ = require('jquery')(window);
+
+var SDPUtil = require('./strophe.jingle.sdp.util.js').SDPUtil;
+
+/**
+ * Returns map of MediaChannel mapped per channel idx.
+ */
+SDP.prototype.getMediaSsrcMap = function() {
+    var self = this;
+    var media_ssrcs = {};
+    for (channelNum = 0; channelNum < self.media.length; channelNum++) {
+        modified = true;
+        tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc:');
+        var type = SDPUtil.parse_mid(SDPUtil.find_line(self.media[channelNum], 'a=mid:'));
+        var channel = new MediaChannel(channelNum, type);
+        media_ssrcs[channelNum] = channel;
+        tmp.forEach(function (line) {
+            var linessrc = line.substring(7).split(' ')[0];
+            // allocate new ChannelSsrc
+            if(!channel.ssrcs[linessrc]) {
+                channel.ssrcs[linessrc] = new ChannelSsrc(linessrc, type);
+            }
+            channel.ssrcs[linessrc].lines.push(line);
+        });
+        tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc-group:');
+        tmp.forEach(function(line){
+            var semantics = line.substr(0, idx).substr(13);
+            var ssrcs = line.substr(14 + semantics.length).split(' ');
+            if (ssrcs.length != 0) {
+                var ssrcGroup = new ChannelSsrcGroup(semantics, ssrcs);
+                channel.ssrcGroups.push(ssrcGroup);
+            }
+        });
+    }
+    return media_ssrcs;
+};
+/**
+ * Returns <tt>true</tt> if this SDP contains given SSRC.
+ * @param ssrc the ssrc to check.
+ * @returns {boolean} <tt>true</tt> if this SDP contains given SSRC.
+ */
+SDP.prototype.containsSSRC = function(ssrc) {
+    var channels = this.getMediaSsrcMap();
+    var contains = false;
+    Object.keys(channels).forEach(function(chNumber){
+        var channel = channels[chNumber];
+        //console.log("Check", channel, ssrc);
+        if(Object.keys(channel.ssrcs).indexOf(ssrc) != -1){
+            contains = true;
+        }
+    });
+    return contains;
+};
+
+/**
+ * Returns map of MediaChannel that contains only media not contained in <tt>otherSdp</tt>. Mapped by channel idx.
+ * @param otherSdp the other SDP to check ssrc with.
+ */
+SDP.prototype.getNewMedia = function(otherSdp) {
+
+    // this could be useful in Array.prototype.
+    function arrayEquals(array) {
+        // if the other array is a falsy value, return
+        if (!array)
+            return false;
+
+        // compare lengths - can save a lot of time
+        if (this.length != array.length)
+            return false;
+
+        for (var i = 0, l=this.length; i < l; i++) {
+            // Check if we have nested arrays
+            if (this[i] instanceof Array && array[i] instanceof Array) {
+                // recurse into the nested arrays
+                if (!this[i].equals(array[i]))
+                    return false;
+            }
+            else if (this[i] != array[i]) {
+                // Warning - two different object instances will never be equal: {x:20} != {x:20}
+                return false;
+            }
+        }
+        return true;
+    }
+
+    var myMedia = this.getMediaSsrcMap();
+    var othersMedia = otherSdp.getMediaSsrcMap();
+    var newMedia = {};
+    Object.keys(othersMedia).forEach(function(channelNum) {
+        var myChannel = myMedia[channelNum];
+        var othersChannel = othersMedia[channelNum];
+        if(!myChannel && othersChannel) {
+            // Add whole channel
+            newMedia[channelNum] = othersChannel;
+            return;
+        }
+        // Look for new ssrcs accross the channel
+        Object.keys(othersChannel.ssrcs).forEach(function(ssrc) {
+            if(Object.keys(myChannel.ssrcs).indexOf(ssrc) === -1) {
+                // Allocate channel if we've found ssrc that doesn't exist in our channel
+                if(!newMedia[channelNum]){
+                    newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
+                }
+                newMedia[channelNum].ssrcs[ssrc] = othersChannel.ssrcs[ssrc];
+            }
+        });
+
+        // Look for new ssrc groups across the channels
+        othersChannel.ssrcGroups.forEach(function(otherSsrcGroup){
+
+            // try to match the other ssrc-group with an ssrc-group of ours
+            var matched = false;
+            for (var i = 0; i < myChannel.ssrcGroups.length; i++) {
+                var mySsrcGroup = myChannel.ssrcGroups[i];
+                if (otherSsrcGroup.semantics == mySsrcGroup.semantics
+                    && arrayEquals.apply(otherSsrcGroup.ssrcs, [mySsrcGroup.ssrcs])) {
+
+                    matched = true;
+                    break;
+                }
+            }
+
+            if (!matched) {
+                // Allocate channel if we've found an ssrc-group that doesn't
+                // exist in our channel
+
+                if(!newMedia[channelNum]){
+                    newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
+                }
+                newMedia[channelNum].ssrcGroups.push(otherSsrcGroup);
+            }
+        });
+    });
+    return newMedia;
+};
+
+// remove iSAC and CN from SDP
+SDP.prototype.mangle = function () {
+    var i, j, mline, lines, rtpmap, newdesc;
+    for (i = 0; i < this.media.length; i++) {
+        lines = this.media[i].split('\r\n');
+        lines.pop(); // remove empty last element
+        mline = SDPUtil.parse_mline(lines.shift());
+        if (mline.media != 'audio')
+            continue;
+        newdesc = '';
+        mline.fmt.length = 0;
+        for (j = 0; j < lines.length; j++) {
+            if (lines[j].substr(0, 9) == 'a=rtpmap:') {
+                rtpmap = SDPUtil.parse_rtpmap(lines[j]);
+                if (rtpmap.name == 'CN' || rtpmap.name == 'ISAC')
+                    continue;
+                mline.fmt.push(rtpmap.id);
+                newdesc += lines[j] + '\r\n';
+            } else {
+                newdesc += lines[j] + '\r\n';
+            }
+        }
+        this.media[i] = SDPUtil.build_mline(mline) + '\r\n';
+        this.media[i] += newdesc;
+    }
+    this.raw = this.session + this.media.join('');
+};
+
+// remove lines matching prefix from session section
+SDP.prototype.removeSessionLines = function(prefix) {
+    var self = this;
+    var lines = SDPUtil.find_lines(this.session, prefix);
+    lines.forEach(function(line) {
+        self.session = self.session.replace(line + '\r\n', '');
+    });
+    this.raw = this.session + this.media.join('');
+    return lines;
+}
+// remove lines matching prefix from a media section specified by mediaindex
+// TODO: non-numeric mediaindex could match mid
+SDP.prototype.removeMediaLines = function(mediaindex, prefix) {
+    var self = this;
+    var lines = SDPUtil.find_lines(this.media[mediaindex], prefix);
+    lines.forEach(function(line) {
+        self.media[mediaindex] = self.media[mediaindex].replace(line + '\r\n', '');
+    });
+    this.raw = this.session + this.media.join('');
+    return lines;
+}
+
+// add content's to a jingle element
+SDP.prototype.toJingle = function (elem, thecreator) {
+    var i, j, k, mline, ssrc, rtpmap, tmp, line, lines;
+    var self = this;
+    // new bundle plan
+    if (SDPUtil.find_line(this.session, 'a=group:')) {
+        lines = SDPUtil.find_lines(this.session, 'a=group:');
+        for (i = 0; i < lines.length; i++) {
+            tmp = lines[i].split(' ');
+            var semantics = tmp.shift().substr(8);
+            elem.c('group', {xmlns: 'urn:xmpp:jingle:apps:grouping:0', semantics:semantics});
+            for (j = 0; j < tmp.length; j++) {
+                elem.c('content', {name: tmp[j]}).up();
+            }
+            elem.up();
+        }
+    }
+    // old bundle plan, to be removed
+    var bundle = [];
+    if (SDPUtil.find_line(this.session, 'a=group:BUNDLE')) {
+        bundle = SDPUtil.find_line(this.session, 'a=group:BUNDLE ').split(' ');
+        bundle.shift();
+    }
+    for (i = 0; i < this.media.length; i++) {
+        mline = SDPUtil.parse_mline(this.media[i].split('\r\n')[0]);
+        if (!(mline.media === 'audio' ||
+              mline.media === 'video' ||
+              mline.media === 'application'))
+        {
+            continue;
+        }
+        if (SDPUtil.find_line(this.media[i], 'a=ssrc:')) {
+            ssrc = SDPUtil.find_line(this.media[i], 'a=ssrc:').substring(7).split(' ')[0]; // take the first
+        } else {
+            ssrc = false;
+        }
+
+        elem.c('content', {creator: thecreator, name: mline.media});
+        if (SDPUtil.find_line(this.media[i], 'a=mid:')) {
+            // prefer identifier from a=mid if present
+            var mid = SDPUtil.parse_mid(SDPUtil.find_line(this.media[i], 'a=mid:'));
+            elem.attrs({ name: mid });
+
+            // old BUNDLE plan, to be removed
+            if (bundle.indexOf(mid) !== -1) {
+                elem.c('bundle', {xmlns: 'http://estos.de/ns/bundle'}).up();
+                bundle.splice(bundle.indexOf(mid), 1);
+            }
+        }
+
+        if (SDPUtil.find_line(this.media[i], 'a=rtpmap:').length)
+        {
+            elem.c('description',
+                {xmlns: 'urn:xmpp:jingle:apps:rtp:1',
+                    media: mline.media });
+            if (ssrc) {
+                elem.attrs({ssrc: ssrc});
+            }
+            for (j = 0; j < mline.fmt.length; j++) {
+                rtpmap = SDPUtil.find_line(this.media[i], 'a=rtpmap:' + mline.fmt[j]);
+                elem.c('payload-type', SDPUtil.parse_rtpmap(rtpmap));
+                // put any 'a=fmtp:' + mline.fmt[j] lines into <param name=foo value=bar/>
+                if (SDPUtil.find_line(this.media[i], 'a=fmtp:' + mline.fmt[j])) {
+                    tmp = SDPUtil.parse_fmtp(SDPUtil.find_line(this.media[i], 'a=fmtp:' + mline.fmt[j]));
+                    for (k = 0; k < tmp.length; k++) {
+                        elem.c('parameter', tmp[k]).up();
+                    }
+                }
+                this.RtcpFbToJingle(i, elem, mline.fmt[j]); // XEP-0293 -- map a=rtcp-fb
+
+                elem.up();
+            }
+            if (SDPUtil.find_line(this.media[i], 'a=crypto:', this.session)) {
+                elem.c('encryption', {required: 1});
+                var crypto = SDPUtil.find_lines(this.media[i], 'a=crypto:', this.session);
+                crypto.forEach(function(line) {
+                    elem.c('crypto', SDPUtil.parse_crypto(line)).up();
+                });
+                elem.up(); // end of encryption
+            }
+
+            if (ssrc) {
+                // new style mapping
+                elem.c('source', { ssrc: ssrc, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                // FIXME: group by ssrc and support multiple different ssrcs
+                var ssrclines = SDPUtil.find_lines(this.media[i], 'a=ssrc:');
+                ssrclines.forEach(function(line) {
+                    idx = line.indexOf(' ');
+                    var linessrc = line.substr(0, idx).substr(7);
+                    if (linessrc != ssrc) {
+                        elem.up();
+                        ssrc = linessrc;
+                        elem.c('source', { ssrc: ssrc, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                    }
+                    var kv = line.substr(idx + 1);
+                    elem.c('parameter');
+                    if (kv.indexOf(':') == -1) {
+                        elem.attrs({ name: kv });
+                    } else {
+                        elem.attrs({ name: kv.split(':', 2)[0] });
+                        elem.attrs({ value: kv.split(':', 2)[1] });
+                    }
+                    elem.up();
+                });
+                elem.up();
+
+                // old proprietary mapping, to be removed at some point
+                tmp = SDPUtil.parse_ssrc(this.media[i]);
+                tmp.xmlns = 'http://estos.de/ns/ssrc';
+                tmp.ssrc = ssrc;
+                elem.c('ssrc', tmp).up(); // ssrc is part of description
+
+                // XEP-0339 handle ssrc-group attributes
+                var ssrc_group_lines = SDPUtil.find_lines(this.media[i], 'a=ssrc-group:');
+                ssrc_group_lines.forEach(function(line) {
+                    idx = line.indexOf(' ');
+                    var semantics = line.substr(0, idx).substr(13);
+                    var ssrcs = line.substr(14 + semantics.length).split(' ');
+                    if (ssrcs.length != 0) {
+                        elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                        ssrcs.forEach(function(ssrc) {
+                            elem.c('source', { ssrc: ssrc })
+                                .up();
+                        });
+                        elem.up();
+                    }
+                });
+            }
+
+            if (SDPUtil.find_line(this.media[i], 'a=rtcp-mux')) {
+                elem.c('rtcp-mux').up();
+            }
+
+            // XEP-0293 -- map a=rtcp-fb:*
+            this.RtcpFbToJingle(i, elem, '*');
+
+            // XEP-0294
+            if (SDPUtil.find_line(this.media[i], 'a=extmap:')) {
+                lines = SDPUtil.find_lines(this.media[i], 'a=extmap:');
+                for (j = 0; j < lines.length; j++) {
+                    tmp = SDPUtil.parse_extmap(lines[j]);
+                    elem.c('rtp-hdrext', { xmlns: 'urn:xmpp:jingle:apps:rtp:rtp-hdrext:0',
+                        uri: tmp.uri,
+                        id: tmp.value });
+                    if (tmp.hasOwnProperty('direction')) {
+                        switch (tmp.direction) {
+                            case 'sendonly':
+                                elem.attrs({senders: 'responder'});
+                                break;
+                            case 'recvonly':
+                                elem.attrs({senders: 'initiator'});
+                                break;
+                            case 'sendrecv':
+                                elem.attrs({senders: 'both'});
+                                break;
+                            case 'inactive':
+                                elem.attrs({senders: 'none'});
+                                break;
+                        }
+                    }
+                    // TODO: handle params
+                    elem.up();
+                }
+            }
+            elem.up(); // end of description
+        }
+
+        // map ice-ufrag/pwd, dtls fingerprint, candidates
+        this.TransportToJingle(i, elem);
+
+        if (SDPUtil.find_line(this.media[i], 'a=sendrecv', this.session)) {
+            elem.attrs({senders: 'both'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=sendonly', this.session)) {
+            elem.attrs({senders: 'initiator'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=recvonly', this.session)) {
+            elem.attrs({senders: 'responder'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=inactive', this.session)) {
+            elem.attrs({senders: 'none'});
+        }
+        if (mline.port == '0') {
+            // estos hack to reject an m-line
+            elem.attrs({senders: 'rejected'});
+        }
+        elem.up(); // end of content
+    }
+    elem.up();
+    return elem;
+};
+
+SDP.prototype.TransportToJingle = function (mediaindex, elem) {
+    var i = mediaindex;
+    var tmp;
+    var self = this;
+    elem.c('transport');
+
+    // XEP-0343 DTLS/SCTP
+    if (SDPUtil.find_line(this.media[mediaindex], 'a=sctpmap:').length)
+    {
+        var sctpmap = SDPUtil.find_line(
+            this.media[i], 'a=sctpmap:', self.session);
+        if (sctpmap)
+        {
+            var sctpAttrs = SDPUtil.parse_sctpmap(sctpmap);
+            elem.c('sctpmap',
+                {
+                    xmlns: 'urn:xmpp:jingle:transports:dtls-sctp:1',
+                    number: sctpAttrs[0], /* SCTP port */
+                    protocol: sctpAttrs[1], /* protocol */
+                });
+            // Optional stream count attribute
+            if (sctpAttrs.length > 2)
+                elem.attrs({ streams: sctpAttrs[2]});
+            elem.up();
+        }
+    }
+    // XEP-0320
+    var fingerprints = SDPUtil.find_lines(this.media[mediaindex], 'a=fingerprint:', this.session);
+    fingerprints.forEach(function(line) {
+        tmp = SDPUtil.parse_fingerprint(line);
+        tmp.xmlns = 'urn:xmpp:jingle:apps:dtls:0';
+        elem.c('fingerprint').t(tmp.fingerprint);
+        delete tmp.fingerprint;
+        line = SDPUtil.find_line(self.media[mediaindex], 'a=setup:', self.session);
+        if (line) {
+            tmp.setup = line.substr(8);
+        }
+        elem.attrs(tmp);
+        elem.up(); // end of fingerprint
+    });
+    tmp = SDPUtil.iceparams(this.media[mediaindex], this.session);
+    if (tmp) {
+        tmp.xmlns = 'urn:xmpp:jingle:transports:ice-udp:1';
+        elem.attrs(tmp);
+        // XEP-0176
+        if (SDPUtil.find_line(this.media[mediaindex], 'a=candidate:', this.session)) { // add any a=candidate lines
+            var lines = SDPUtil.find_lines(this.media[mediaindex], 'a=candidate:', this.session);
+            lines.forEach(function (line) {
+                elem.c('candidate', SDPUtil.candidateToJingle(line)).up();
+            });
+        }
+    }
+    elem.up(); // end of transport
+}
+
+SDP.prototype.RtcpFbToJingle = function (mediaindex, elem, payloadtype) { // XEP-0293
+    var lines = SDPUtil.find_lines(this.media[mediaindex], 'a=rtcp-fb:' + payloadtype);
+    lines.forEach(function (line) {
+        var tmp = SDPUtil.parse_rtcpfb(line);
+        if (tmp.type == 'trr-int') {
+            elem.c('rtcp-fb-trr-int', {xmlns: 'urn:xmpp:jingle:apps:rtp:rtcp-fb:0', value: tmp.params[0]});
+            elem.up();
+        } else {
+            elem.c('rtcp-fb', {xmlns: 'urn:xmpp:jingle:apps:rtp:rtcp-fb:0', type: tmp.type});
+            if (tmp.params.length > 0) {
+                elem.attrs({'subtype': tmp.params[0]});
+            }
+            elem.up();
+        }
+    });
+};
+
+SDP.prototype.RtcpFbFromJingle = function (elem, payloadtype) { // XEP-0293
+    var media = '';
+    var tmp = elem.find('>rtcp-fb-trr-int[xmlns="urn:xmpp:jingle:apps:rtp:rtcp-fb:0"]');
+    if (tmp.length) {
+        media += 'a=rtcp-fb:' + '*' + ' ' + 'trr-int' + ' ';
+        if (tmp.attr('value')) {
+            media += tmp.attr('value');
+        } else {
+            media += '0';
+        }
+        media += '\r\n';
+    }
+    tmp = elem.find('>rtcp-fb[xmlns="urn:xmpp:jingle:apps:rtp:rtcp-fb:0"]');
+    tmp.each(function () {
+        media += 'a=rtcp-fb:' + payloadtype + ' ' + $(this).attr('type');
+        if ($(this).attr('subtype')) {
+            media += ' ' + $(this).attr('subtype');
+        }
+        media += '\r\n';
+    });
+    return media;
+};
+
+// construct an SDP from a jingle stanza
+SDP.prototype.fromJingle = function (jingle) {
+    var self = this;
+    this.raw = 'v=0\r\n' +
+        'o=- ' + '1923518516' + ' 2 IN IP4 0.0.0.0\r\n' +// FIXME
+        's=-\r\n' +
+        't=0 0\r\n';
+    // http://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-04#section-8
+    if ($(jingle).find('>group[xmlns="urn:xmpp:jingle:apps:grouping:0"]').length) {
+        $(jingle).find('>group[xmlns="urn:xmpp:jingle:apps:grouping:0"]').each(function (idx, group) {
+            var contents = $(group).find('>content').map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+            if (contents.length > 0) {
+                self.raw += 'a=group:' + (group.getAttribute('semantics') || group.getAttribute('type')) + ' ' + contents.join(' ') + '\r\n';
+            }
+        });
+    } else if ($(jingle).find('>group[xmlns="urn:ietf:rfc:5888"]').length) {
+        // temporary namespace, not to be used. to be removed soon.
+        $(jingle).find('>group[xmlns="urn:ietf:rfc:5888"]').each(function (idx, group) {
+            var contents = $(group).find('>content').map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+            if (group.getAttribute('type') !== null && contents.length > 0) {
+                self.raw += 'a=group:' + group.getAttribute('type') + ' ' + contents.join(' ') + '\r\n';
+            }
+        });
+    } else {
+        // for backward compability, to be removed soon
+        // assume all contents are in the same bundle group, can be improved upon later
+        var bundle = $(jingle).find('>content').filter(function (idx, content) {
+            //elem.c('bundle', {xmlns:'http://estos.de/ns/bundle'});
+            return $(content).find('>bundle').length > 0;
+        }).map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+        if (bundle.length) {
+            this.raw += 'a=group:BUNDLE ' + bundle.join(' ') + '\r\n';
+        }
+    }
+
+    this.session = this.raw;
+    jingle.find('>content').each(function () {
+        var m = self.jingle2media($(this));
+        self.media.push(m);
+    });
+
+    // reconstruct msid-semantic -- apparently not necessary
+    /*
+     var msid = SDPUtil.parse_ssrc(this.raw);
+     if (msid.hasOwnProperty('mslabel')) {
+     this.session += "a=msid-semantic: WMS " + msid.mslabel + "\r\n";
+     }
+     */
+
+    this.raw = this.session + this.media.join('');
+};
+
+// translate a jingle content element into an an SDP media part
+SDP.prototype.jingle2media = function (content) {
+    var media = '',
+        desc = content.find('description'),
+        ssrc = desc.attr('ssrc'),
+        self = this,
+        tmp;
+    var sctp = content.find(
+        '>transport>sctpmap[xmlns="urn:xmpp:jingle:transports:dtls-sctp:1"]');
+
+    tmp = { media: desc.attr('media') };
+    tmp.port = '1';
+    if (content.attr('senders') == 'rejected') {
+        // estos hack to reject an m-line.
+        tmp.port = '0';
+    }
+    if (content.find('>transport>fingerprint').length || desc.find('encryption').length) {
+        if (sctp.length)
+            tmp.proto = 'DTLS/SCTP';
+        else
+            tmp.proto = 'RTP/SAVPF';
+    } else {
+        tmp.proto = 'RTP/AVPF';
+    }
+    if (!sctp.length)
+    {
+        tmp.fmt = desc.find('payload-type').map(
+            function () { return this.getAttribute('id'); }).get();
+        media += SDPUtil.build_mline(tmp) + '\r\n';
+    }
+    else
+    {
+        media += 'm=application 1 DTLS/SCTP ' + sctp.attr('number') + '\r\n';
+        media += 'a=sctpmap:' + sctp.attr('number') +
+            ' ' + sctp.attr('protocol');
+
+        var streamCount = sctp.attr('streams');
+        if (streamCount)
+            media += ' ' + streamCount + '\r\n';
+        else
+            media += '\r\n';
+    }
+
+    media += 'c=IN IP4 0.0.0.0\r\n';
+    if (!sctp.length)
+        media += 'a=rtcp:1 IN IP4 0.0.0.0\r\n';
+    //tmp = content.find('>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]');
+    tmp = content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]');
+    //console.log('transports: '+content.find('>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]').length);
+    //console.log('bundle.transports: '+content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]').length);
+    //console.log("tmp fingerprint: "+tmp.find('>fingerprint').innerHTML);
+    if (tmp.length) {
+        if (tmp.attr('ufrag')) {
+            media += SDPUtil.build_iceufrag(tmp.attr('ufrag')) + '\r\n';
+        }
+        if (tmp.attr('pwd')) {
+            media += SDPUtil.build_icepwd(tmp.attr('pwd')) + '\r\n';
+        }
+        tmp.find('>fingerprint').each(function () {
+            // FIXME: check namespace at some point
+            media += 'a=fingerprint:' + this.getAttribute('hash');
+            media += ' ' + $(this).text();
+            media += '\r\n';
+            //console.log("mline "+media);
+            if (this.getAttribute('setup')) {
+                media += 'a=setup:' + this.getAttribute('setup') + '\r\n';
+            }
+        });
+    }
+    switch (content.attr('senders')) {
+        case 'initiator':
+            media += 'a=sendonly\r\n';
+            break;
+        case 'responder':
+            media += 'a=recvonly\r\n';
+            break;
+        case 'none':
+            media += 'a=inactive\r\n';
+            break;
+        case 'both':
+            media += 'a=sendrecv\r\n';
+            break;
+    }
+    media += 'a=mid:' + content.attr('name') + '\r\n';
+    /*if (content.attr('name') == 'video') {
+        media += 'a=x-google-flag:conference' + '\r\n';
+    }*/
+
+    // <description><rtcp-mux/></description>
+    // see http://code.google.com/p/libjingle/issues/detail?id=309 -- no spec though
+    // and http://mail.jabber.org/pipermail/jingle/2011-December/001761.html
+    if (desc.find('rtcp-mux').length) {
+        media += 'a=rtcp-mux\r\n';
+    }
+
+    if (desc.find('encryption').length) {
+        desc.find('encryption>crypto').each(function () {
+            media += 'a=crypto:' + this.getAttribute('tag');
+            media += ' ' + this.getAttribute('crypto-suite');
+            media += ' ' + this.getAttribute('key-params');
+            if (this.getAttribute('session-params')) {
+                media += ' ' + this.getAttribute('session-params');
+            }
+            media += '\r\n';
+        });
+    }
+    desc.find('payload-type').each(function () {
+        media += SDPUtil.build_rtpmap(this) + '\r\n';
+        if ($(this).find('>parameter').length) {
+            media += 'a=fmtp:' + this.getAttribute('id') + ' ';
+            media += $(this).find('parameter').map(function () { return (this.getAttribute('name') ? (this.getAttribute('name') + '=') : '') + this.getAttribute('value'); }).get().join('; ');
+            media += '\r\n';
+        }
+        // xep-0293
+        media += self.RtcpFbFromJingle($(this), this.getAttribute('id'));
+    });
+
+    // xep-0293
+    media += self.RtcpFbFromJingle(desc, '*');
+
+    // xep-0294
+    tmp = desc.find('>rtp-hdrext[xmlns="urn:xmpp:jingle:apps:rtp:rtp-hdrext:0"]');
+    tmp.each(function () {
+        media += 'a=extmap:' + this.getAttribute('id') + ' ' + this.getAttribute('uri') + '\r\n';
+    });
+
+    content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]>candidate').each(function () {
+        media += SDPUtil.candidateFromJingle(this);
+    });
+
+    // XEP-0339 handle ssrc-group attributes
+    tmp = content.find('description>ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
+        var semantics = this.getAttribute('semantics');
+        var ssrcs = $(this).find('>source').map(function() {
+            return this.getAttribute('ssrc');
+        }).get();
+
+        if (ssrcs.length != 0) {
+            media += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
+        }
+    });
+
+    tmp = content.find('description>source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]');
+    tmp.each(function () {
+        var ssrc = this.getAttribute('ssrc');
+        $(this).find('>parameter').each(function () {
+            media += 'a=ssrc:' + ssrc + ' ' + this.getAttribute('name');
+            if (this.getAttribute('value') && this.getAttribute('value').length)
+                media += ':' + this.getAttribute('value');
+            media += '\r\n';
+        });
+    });
+
+    if (tmp.length === 0) {
+        // fallback to proprietary mapping of a=ssrc lines
+        tmp = content.find('description>ssrc[xmlns="http://estos.de/ns/ssrc"]');
+        if (tmp.length) {
+            media += 'a=ssrc:' + ssrc + ' cname:' + tmp.attr('cname') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' msid:' + tmp.attr('msid') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' mslabel:' + tmp.attr('mslabel') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' label:' + tmp.attr('label') + '\r\n';
+        }
+    }
+    return media;
+};
+

contrib/jitsimeetbridge/unjingle/strophe.jingle.sdp.util.js

@@ -0,0 +1,408 @@
+/**
+ * Contains utility classes used in SDP class.
+ *
+ */
+
+/**
+ * Class holds a=ssrc lines and media type a=mid
+ * @param ssrc synchronization source identifier number(a=ssrc lines from SDP)
+ * @param type media type eg. "audio" or "video"(a=mid frm SDP)
+ * @constructor
+ */
+function ChannelSsrc(ssrc, type) {
+    this.ssrc = ssrc;
+    this.type = type;
+    this.lines = [];
+}
+
+/**
+ * Class holds a=ssrc-group: lines
+ * @param semantics
+ * @param ssrcs
+ * @constructor
+ */
+function ChannelSsrcGroup(semantics, ssrcs, line) {
+    this.semantics = semantics;
+    this.ssrcs = ssrcs;
+}
+
+/**
+ * Helper class represents media channel. Is a container for ChannelSsrc, holds channel idx and media type.
+ * @param channelNumber channel idx in SDP media array.
+ * @param mediaType media type(a=mid)
+ * @constructor
+ */
+function MediaChannel(channelNumber, mediaType) {
+    /**
+     * SDP channel number
+     * @type {*}
+     */
+    this.chNumber = channelNumber;
+    /**
+     * Channel media type(a=mid)
+     * @type {*}
+     */
+    this.mediaType = mediaType;
+    /**
+     * The maps of ssrc numbers to ChannelSsrc objects.
+     */
+    this.ssrcs = {};
+
+    /**
+     * The array of ChannelSsrcGroup objects.
+     * @type {Array}
+     */
+    this.ssrcGroups = [];
+}
+
+SDPUtil = {
+    iceparams: function (mediadesc, sessiondesc) {
+        var data = null;
+        if (SDPUtil.find_line(mediadesc, 'a=ice-ufrag:', sessiondesc) &&
+            SDPUtil.find_line(mediadesc, 'a=ice-pwd:', sessiondesc)) {
+            data = {
+                ufrag: SDPUtil.parse_iceufrag(SDPUtil.find_line(mediadesc, 'a=ice-ufrag:', sessiondesc)),
+                pwd: SDPUtil.parse_icepwd(SDPUtil.find_line(mediadesc, 'a=ice-pwd:', sessiondesc))
+            };
+        }
+        return data;
+    },
+    parse_iceufrag: function (line) {
+        return line.substring(12);
+    },
+    build_iceufrag: function (frag) {
+        return 'a=ice-ufrag:' + frag;
+    },
+    parse_icepwd: function (line) {
+        return line.substring(10);
+    },
+    build_icepwd: function (pwd) {
+        return 'a=ice-pwd:' + pwd;
+    },
+    parse_mid: function (line) {
+        return line.substring(6);
+    },
+    parse_mline: function (line) {
+        var parts = line.substring(2).split(' '),
+            data = {};
+        data.media = parts.shift();
+        data.port = parts.shift();
+        data.proto = parts.shift();
+        if (parts[parts.length - 1] === '') { // trailing whitespace
+            parts.pop();
+        }
+        data.fmt = parts;
+        return data;
+    },
+    build_mline: function (mline) {
+        return 'm=' + mline.media + ' ' + mline.port + ' ' + mline.proto + ' ' + mline.fmt.join(' ');
+    },
+    parse_rtpmap: function (line) {
+        var parts = line.substring(9).split(' '),
+            data = {};
+        data.id = parts.shift();
+        parts = parts[0].split('/');
+        data.name = parts.shift();
+        data.clockrate = parts.shift();
+        data.channels = parts.length ? parts.shift() : '1';
+        return data;
+    },
+    /**
+     * Parses SDP line "a=sctpmap:..." and extracts SCTP port from it.
+     * @param line eg. "a=sctpmap:5000 webrtc-datachannel"
+     * @returns [SCTP port number, protocol, streams]
+     */
+    parse_sctpmap: function (line)
+    {
+        var parts = line.substring(10).split(' ');
+        var sctpPort = parts[0];
+        var protocol = parts[1];
+        // Stream count is optional
+        var streamCount = parts.length > 2 ? parts[2] : null;
+        return [sctpPort, protocol, streamCount];// SCTP port
+    },
+    build_rtpmap: function (el) {
+        var line = 'a=rtpmap:' + el.getAttribute('id') + ' ' + el.getAttribute('name') + '/' + el.getAttribute('clockrate');
+        if (el.getAttribute('channels') && el.getAttribute('channels') != '1') {
+            line += '/' + el.getAttribute('channels');
+        }
+        return line;
+    },
+    parse_crypto: function (line) {
+        var parts = line.substring(9).split(' '),
+            data = {};
+        data.tag = parts.shift();
+        data['crypto-suite'] = parts.shift();
+        data['key-params'] = parts.shift();
+        if (parts.length) {
+            data['session-params'] = parts.join(' ');
+        }
+        return data;
+    },
+    parse_fingerprint: function (line) { // RFC 4572
+        var parts = line.substring(14).split(' '),
+            data = {};
+        data.hash = parts.shift();
+        data.fingerprint = parts.shift();
+        // TODO assert that fingerprint satisfies 2UHEX *(":" 2UHEX) ?
+        return data;
+    },
+    parse_fmtp: function (line) {
+        var parts = line.split(' '),
+            i, key, value,
+            data = [];
+        parts.shift();
+        parts = parts.join(' ').split(';');
+        for (i = 0; i < parts.length; i++) {
+            key = parts[i].split('=')[0];
+            while (key.length && key[0] == ' ') {
+                key = key.substring(1);
+            }
+            value = parts[i].split('=')[1];
+            if (key && value) {
+                data.push({name: key, value: value});
+            } else if (key) {
+                // rfc 4733 (DTMF) style stuff
+                data.push({name: '', value: key});
+            }
+        }
+        return data;
+    },
+    parse_icecandidate: function (line) {
+        var candidate = {},
+            elems = line.split(' ');
+        candidate.foundation = elems[0].substring(12);
+        candidate.component = elems[1];
+        candidate.protocol = elems[2].toLowerCase();
+        candidate.priority = elems[3];
+        candidate.ip = elems[4];
+        candidate.port = elems[5];
+        // elems[6] => "typ"
+        candidate.type = elems[7];
+        candidate.generation = 0; // default value, may be overwritten below
+        for (var i = 8; i < elems.length; i += 2) {
+            switch (elems[i]) {
+                case 'raddr':
+                    candidate['rel-addr'] = elems[i + 1];
+                    break;
+                case 'rport':
+                    candidate['rel-port'] = elems[i + 1];
+                    break;
+                case 'generation':
+                    candidate.generation = elems[i + 1];
+                    break;
+                case 'tcptype':
+                    candidate.tcptype = elems[i + 1];
+                    break;
+                default: // TODO
+                    console.log('parse_icecandidate not translating "' + elems[i] + '" = "' + elems[i + 1] + '"');
+            }
+        }
+        candidate.network = '1';
+        candidate.id = Math.random().toString(36).substr(2, 10); // not applicable to SDP -- FIXME: should be unique, not just random
+        return candidate;
+    },
+    build_icecandidate: function (cand) {
+        var line = ['a=candidate:' + cand.foundation, cand.component, cand.protocol, cand.priority, cand.ip, cand.port, 'typ', cand.type].join(' ');
+        line += ' ';
+        switch (cand.type) {
+            case 'srflx':
+            case 'prflx':
+            case 'relay':
+                if (cand.hasOwnAttribute('rel-addr') && cand.hasOwnAttribute('rel-port')) {
+                    line += 'raddr';
+                    line += ' ';
+                    line += cand['rel-addr'];
+                    line += ' ';
+                    line += 'rport';
+                    line += ' ';
+                    line += cand['rel-port'];
+                    line += ' ';
+                }
+                break;
+        }
+        if (cand.hasOwnAttribute('tcptype')) {
+            line += 'tcptype';
+            line += ' ';
+            line += cand.tcptype;
+            line += ' ';
+        }
+        line += 'generation';
+        line += ' ';
+        line += cand.hasOwnAttribute('generation') ? cand.generation : '0';
+        return line;
+    },
+    parse_ssrc: function (desc) {
+        // proprietary mapping of a=ssrc lines
+        // TODO: see "Jingle RTP Source Description" by Juberti and P. Thatcher on google docs
+        // and parse according to that
+        var lines = desc.split('\r\n'),
+            data = {};
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, 7) == 'a=ssrc:') {
+                var idx = lines[i].indexOf(' ');
+                data[lines[i].substr(idx + 1).split(':', 2)[0]] = lines[i].substr(idx + 1).split(':', 2)[1];
+            }
+        }
+        return data;
+    },
+    parse_rtcpfb: function (line) {
+        var parts = line.substr(10).split(' ');
+        var data = {};
+        data.pt = parts.shift();
+        data.type = parts.shift();
+        data.params = parts;
+        return data;
+    },
+    parse_extmap: function (line) {
+        var parts = line.substr(9).split(' ');
+        var data = {};
+        data.value = parts.shift();
+        if (data.value.indexOf('/') != -1) {
+            data.direction = data.value.substr(data.value.indexOf('/') + 1);
+            data.value = data.value.substr(0, data.value.indexOf('/'));
+        } else {
+            data.direction = 'both';
+        }
+        data.uri = parts.shift();
+        data.params = parts;
+        return data;
+    },
+    find_line: function (haystack, needle, sessionpart) {
+        var lines = haystack.split('\r\n');
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, needle.length) == needle) {
+                return lines[i];
+            }
+        }
+        if (!sessionpart) {
+            return false;
+        }
+        // search session part
+        lines = sessionpart.split('\r\n');
+        for (var j = 0; j < lines.length; j++) {
+            if (lines[j].substring(0, needle.length) == needle) {
+                return lines[j];
+            }
+        }
+        return false;
+    },
+    find_lines: function (haystack, needle, sessionpart) {
+        var lines = haystack.split('\r\n'),
+            needles = [];
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, needle.length) == needle)
+                needles.push(lines[i]);
+        }
+        if (needles.length || !sessionpart) {
+            return needles;
+        }
+        // search session part
+        lines = sessionpart.split('\r\n');
+        for (var j = 0; j < lines.length; j++) {
+            if (lines[j].substring(0, needle.length) == needle) {
+                needles.push(lines[j]);
+            }
+        }
+        return needles;
+    },
+    candidateToJingle: function (line) {
+        // a=candidate:2979166662 1 udp 2113937151 192.168.2.100 57698 typ host generation 0
+        //      <candidate component=... foundation=... generation=... id=... ip=... network=... port=... priority=... protocol=... type=.../>
+        if (line.indexOf('candidate:') === 0) {
+            line = 'a=' + line;
+        } else if (line.substring(0, 12) != 'a=candidate:') {
+            console.log('parseCandidate called with a line that is not a candidate line');
+            console.log(line);
+            return null;
+        }
+        if (line.substring(line.length - 2) == '\r\n') // chomp it
+            line = line.substring(0, line.length - 2);
+        var candidate = {},
+            elems = line.split(' '),
+            i;
+        if (elems[6] != 'typ') {
+            console.log('did not find typ in the right place');
+            console.log(line);
+            return null;
+        }
+        candidate.foundation = elems[0].substring(12);
+        candidate.component = elems[1];
+        candidate.protocol = elems[2].toLowerCase();
+        candidate.priority = elems[3];
+        candidate.ip = elems[4];
+        candidate.port = elems[5];
+        // elems[6] => "typ"
+        candidate.type = elems[7];
+
+        candidate.generation = '0'; // default, may be overwritten below
+        for (i = 8; i < elems.length; i += 2) {
+            switch (elems[i]) {
+                case 'raddr':
+                    candidate['rel-addr'] = elems[i + 1];
+                    break;
+                case 'rport':
+                    candidate['rel-port'] = elems[i + 1];
+                    break;
+                case 'generation':
+                    candidate.generation = elems[i + 1];
+                    break;
+                case 'tcptype':
+                    candidate.tcptype = elems[i + 1];
+                    break;
+                default: // TODO
+                    console.log('not translating "' + elems[i] + '" = "' + elems[i + 1] + '"');
+            }
+        }
+        candidate.network = '1';
+        candidate.id = Math.random().toString(36).substr(2, 10); // not applicable to SDP -- FIXME: should be unique, not just random
+        return candidate;
+    },
+    candidateFromJingle: function (cand) {
+        var line = 'a=candidate:';
+        line += cand.getAttribute('foundation');
+        line += ' ';
+        line += cand.getAttribute('component');
+        line += ' ';
+        line += cand.getAttribute('protocol'); //.toUpperCase(); // chrome M23 doesn't like this
+        line += ' ';
+        line += cand.getAttribute('priority');
+        line += ' ';
+        line += cand.getAttribute('ip');
+        line += ' ';
+        line += cand.getAttribute('port');
+        line += ' ';
+        line += 'typ';
+        line += ' ' + cand.getAttribute('type');
+        line += ' ';
+        switch (cand.getAttribute('type')) {
+            case 'srflx':
+            case 'prflx':
+            case 'relay':
+                if (cand.getAttribute('rel-addr') && cand.getAttribute('rel-port')) {
+                    line += 'raddr';
+                    line += ' ';
+                    line += cand.getAttribute('rel-addr');
+                    line += ' ';
+                    line += 'rport';
+                    line += ' ';
+                    line += cand.getAttribute('rel-port');
+                    line += ' ';
+                }
+                break;
+        }
+        if (cand.getAttribute('protocol').toLowerCase() == 'tcp') {
+            line += 'tcptype';
+            line += ' ';
+            line += cand.getAttribute('tcptype');
+            line += ' ';
+        }
+        line += 'generation';
+        line += ' ';
+        line += cand.getAttribute('generation') || '0';
+        return line + '\r\n';
+    }
+};
+
+exports.SDPUtil = SDPUtil;
+