Merge branch 'babolivier/email_config' into dinsic

When enabling the account validity feature, Synapse will look at startup for registered account without an expiration date, and will set one equals to 'now + validity_period' for them. On large servers, it can mean that a large number of users will have the same expiration date, which means that they will all be sent a renewal email at the same time, which isn't ideal.
In order to mitigate this, this PR allows server admins to define a 'max_delta' so that the expiration date is a random value in the [now + validity_period ; now + validity_period + max_delta] range. This allows renewal emails to be progressively sent over a configured period instead of being sent all in one big batch.

.buildkite/pipeline.yml

@@ -26,16 +26,6 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
-  - command:
-      - "python -m pip install tox"
-      - "scripts-dev/check-newsfragment"
-    label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-*"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-          propagate-environment: true
-
   - wait
 
   - command:

MANIFEST.in

@@ -43,3 +43,8 @@ prune .buildkite
 
 exclude jenkins*
 recursive-exclude jenkins *.sh
+
+# FIXME: we shouldn't have these templates here
+recursive-include res/templates-dinsic *.css
+recursive-include res/templates-dinsic *.html
+recursive-include res/templates-dinsic *.txt

changelog.d/5083.feature

@@ -0,0 +1 @@
+Adds auth_profile_reqs option to require access_token to GET /profile endpoints on CS API

changelog.d/5098.misc

@@ -0,0 +1 @@
+Add workarounds for pep-517 install errors.

changelog.d/5214.feature

@@ -0,0 +1 @@
+Allow server admins to define and enforce a password policy (MSC2000).

changelog.d/5276.feature

@@ -0,0 +1 @@
+Allow configuring a range for the account validity startup job.

docs/sample_config.yaml

@@ -753,7 +753,9 @@ uploads_path: "DATADIR/uploads"
 # This means that, if a validity period is set, and Synapse is restarted (it will
 # then derive an expiration date from the current validity period), and some time
 # after that the validity period changes and Synapse is restarted, the users'
-# expiration dates won't be updated unless their account is manually renewed.
+# expiration dates won't be updated unless their account is manually renewed. This
+# date will be randomly selected within a range [now + period - d ; now + period],
+# where d is equal to 10% of the validity period.
 #
 #account_validity:
 #  enabled: True
@@ -772,9 +774,32 @@ uploads_path: "DATADIR/uploads"
 #
 #disable_msisdn_registration: true
 
+# Derive the user's matrix ID from a type of 3PID used when registering.
+# This overrides any matrix ID the user proposes when calling /register
+# The 3PID type should be present in registrations_require_3pid to avoid
+# users failing to register if they don't specify the right kind of 3pid.
+#
+#register_mxid_from_3pid: email
+
+# Uncomment to set the display name of new users to their email address,
+# rather than using the default heuristic.
+#
+#register_just_use_email_for_display_name: true
+
 # Mandate that users are only allowed to associate certain formats of
 # 3PIDs with accounts on this server.
 #
+# Use an Identity Server to establish which 3PIDs are allowed to register?
+# Overrides allowed_local_3pids below.
+#
+#check_is_for_allowed_local_3pids: matrix.org
+#
+# If you are using an IS you can also check whether that IS registers
+# pending invites for the given 3PID (and then allow it to sign up on
+# the platform):
+#
+#allow_invited_3pids: False
+#
 #allowed_local_3pids:
 #  - medium: email
 #    pattern: '.*@matrix\.org'
@@ -783,6 +808,11 @@ uploads_path: "DATADIR/uploads"
 #  - medium: msisdn
 #    pattern: '\+44'
 
+# If true, stop users from trying to change the 3PIDs associated with
+# their accounts.
+#
+#disable_3pid_changes: False
+
 # Enable 3PIDs lookup requests to identity servers from this server.
 #
 #enable_3pid_lookup: true
@@ -824,6 +854,30 @@ uploads_path: "DATADIR/uploads"
 #  - matrix.org
 #  - vector.im
 
+# If enabled, user IDs, display names and avatar URLs will be replicated
+# to this server whenever they change.
+# This is an experimental API currently implemented by sydent to support
+# cross-homeserver user directories.
+#
+#replicate_user_profiles_to: example.com
+
+# If specified, attempt to replay registrations, profile changes & 3pid
+# bindings on the given target homeserver via the AS API. The HS is authed
+# via a given AS token.
+#
+#shadow_server:
+#  hs_url: https://shadow.example.com
+#  hs: shadow.example.com
+#  as_token: 12u394refgbdhivsia
+
+# If enabled, don't let users set their own display names/avatars
+# other than for the very first time (unless they are a server admin).
+# Useful when provisioning users based on the contents of a 3rd party
+# directory and to avoid ambiguities.
+#
+#disable_set_displayname: False
+#disable_set_avatar_url: False
+
 # Users who register on this homeserver will automatically be joined
 # to these rooms
 #
@@ -1004,6 +1058,36 @@ password_config:
    #
    #pepper: "EVEN_MORE_SECRET"
 
+   # Define and enforce a password policy. Each parameter is optional, boolean
+   # parameters default to 'false' and integer parameters default to 0.
+   # This is an early implementation of MSC2000.
+   #
+   #policy:
+      # Whether to enforce the password policy.
+      #
+      #enabled: true
+
+      # Minimum accepted length for a password.
+      #
+      #minimum_length: 15
+
+      # Whether a password must contain at least one digit.
+      #
+      #require_digit: true
+
+      # Whether a password must contain at least one symbol.
+      # A symbol is any character that's not a number or a letter.
+      #
+      #require_symbol: true
+
+      # Whether a password must contain at least one lowercase letter.
+      #
+      #require_lowercase: true
+
+      # Whether a password must contain at least one lowercase letter.
+      #
+      #require_uppercase: true
+
 
 
 # Enable sending emails for notification events or expiry notices
@@ -1100,6 +1184,11 @@ password_config:
 #user_directory:
 #  enabled: true
 #  search_all_users: false
+#
+#  # If this is set, user search will be delegated to this ID server instead
+#  # of synapse performing the search itself.
+#  # This is an experimental API.
+#  defer_to_id_server: https://id.example.com
 
 
 # User Consent configuration

res/templates-dinsic/mail-Vector.css

@@ -0,0 +1,7 @@
+.header {
+    border-bottom: 4px solid #e4f7ed ! important;
+}
+
+.notif_link a, .footer a {
+    color: #76CFA6 ! important;
+}

res/templates-dinsic/mail.css

@@ -0,0 +1,156 @@
+body {
+    margin: 0px;
+}
+
+pre, code {
+    word-break: break-word;
+    white-space: pre-wrap;
+}
+
+#page {
+    font-family: 'Open Sans', Helvetica, Arial, Sans-Serif;
+    font-color: #454545;
+    font-size: 12pt;
+    width: 100%;
+    padding: 20px;
+}
+
+#inner {
+    width: 640px;
+}
+
+.header {
+    width: 100%;
+    height: 87px;
+    color: #454545;
+    border-bottom: 4px solid #e5e5e5;
+}
+
+.logo {
+    text-align: right;
+    margin-left: 20px;
+}
+
+.salutation {
+    padding-top: 10px;
+    font-weight: bold;
+}
+
+.summarytext {
+}
+
+.room {
+    width: 100%;
+    color: #454545;
+    border-bottom: 1px solid #e5e5e5;
+}
+
+.room_header td {
+    padding-top: 38px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #e5e5e5;
+}
+
+.room_name {
+    vertical-align: middle;
+    font-size: 18px;
+    font-weight: bold;
+}
+
+.room_header h2 {
+    margin-top: 0px;
+    margin-left: 75px;
+    font-size: 20px;
+}
+
+.room_avatar {
+    width: 56px;
+    line-height: 0px;
+    text-align: center;
+    vertical-align: middle;
+}
+
+.room_avatar img {
+    width: 48px;
+    height: 48px;
+    object-fit: cover;
+    border-radius: 24px;
+}
+
+.notif {
+    border-bottom: 1px solid #e5e5e5;
+    margin-top: 16px;
+    padding-bottom: 16px;
+}
+
+.historical_message .sender_avatar {
+    opacity: 0.3;
+}
+
+/* spell out opacity and historical_message class names for Outlook aka Word */
+.historical_message .sender_name {
+    color: #e3e3e3;
+}
+
+.historical_message .message_time {
+    color: #e3e3e3;
+}
+
+.historical_message .message_body {
+    color: #c7c7c7;
+}
+
+.historical_message td,
+.message td {
+    padding-top: 10px;
+}
+
+.sender_avatar {
+    width: 56px;
+    text-align: center;
+    vertical-align: top;
+}
+
+.sender_avatar img {
+    margin-top: -2px;
+    width: 32px;
+    height: 32px;
+    border-radius: 16px;
+}
+
+.sender_name  {
+    display: inline;
+    font-size: 13px;
+    color: #a2a2a2;
+}
+
+.message_time  {
+    text-align: right;
+    width: 100px;
+    font-size: 11px;
+    color: #a2a2a2;
+}
+
+.message_body {
+}
+
+.notif_link td {
+    padding-top: 10px;
+    padding-bottom: 10px;
+    font-weight: bold;
+}
+
+.notif_link a, .footer a {
+    color: #454545;
+    text-decoration: none;
+}
+
+.debug {
+    font-size: 10px;
+    color: #888;
+}
+
+.footer {
+    margin-top: 20px;
+    text-align: center;
+}

res/templates-dinsic/notif.html

@@ -0,0 +1,45 @@
+{% for message in notif.messages %}
+    <tr class="{{ "historical_message" if message.is_historical else "message" }}">
+        <td class="sender_avatar">
+            {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
+                {% if message.sender_avatar_url %}
+                    <img alt="" class="sender_avatar" src="{{ message.sender_avatar_url|mxc_to_http(32,32) }}"  />
+                {% else %}
+                    {% if message.sender_hash % 3 == 0 %}
+                        <img class="sender_avatar" src="https://vector.im/beta/img/76cfa6.png"  />
+                    {% elif message.sender_hash % 3 == 1 %}
+                        <img class="sender_avatar" src="https://vector.im/beta/img/50e2c2.png"  />
+                    {% else %}
+                        <img class="sender_avatar" src="https://vector.im/beta/img/f4c371.png"  />
+                    {% endif %}
+                {% endif %}
+            {% endif %}
+        </td>
+        <td class="message_contents">
+            {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
+                <div class="sender_name">{% if message.msgtype == "m.emote" %}*{% endif %} {{ message.sender_name }}</div>
+            {% endif %}
+            <div class="message_body">
+                {% if message.msgtype == "m.text" %}
+                    {{ message.body_text_html }}
+                {% elif message.msgtype == "m.emote" %}
+                    {{ message.body_text_html }}
+                {% elif message.msgtype == "m.notice" %}
+                    {{ message.body_text_html }}
+                {% elif message.msgtype == "m.image" %}
+                    <img src="{{ message.image_url|mxc_to_http(640, 480, scale) }}" />
+                {% elif message.msgtype == "m.file" %}
+                    <span class="filename">{{ message.body_text_plain }}</span>
+                {% endif %}
+            </div>
+        </td>
+        <td class="message_time">{{ message.ts|format_ts("%H:%M") }}</td>
+    </tr>
+{% endfor %}
+<tr class="notif_link">
+    <td></td>
+    <td>
+        <a href="{{ notif.link }}">Voir {{ room.title }}</a>
+    </td>
+    <td></td>
+</tr>

res/templates-dinsic/notif.txt

@@ -0,0 +1,16 @@
+{% for message in notif.messages %}
+{% if message.msgtype == "m.emote" %}* {% endif %}{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }})
+{% if message.msgtype == "m.text" %}
+{{ message.body_text_plain }}
+{% elif message.msgtype == "m.emote" %}
+{{ message.body_text_plain }}
+{% elif message.msgtype == "m.notice" %}
+{{ message.body_text_plain }}
+{% elif message.msgtype == "m.image" %}
+{{ message.body_text_plain }}
+{% elif message.msgtype == "m.file" %}
+{{ message.body_text_plain }}
+{% endif %}
+{% endfor %}
+
+Voir {{ room.title }} à {{ notif.link }}

res/templates-dinsic/notif_mail.html

@@ -0,0 +1,55 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <style type="text/css">
+            {% include 'mail.css' without context %}
+            {% include "mail-%s.css" % app_name ignore missing without context %}
+        </style>
+    </head>
+    <body>
+        <table id="page">
+            <tr>
+                <td> </td>
+                <td id="inner">
+                    <table class="header">
+                        <tr>
+                            <td>
+                                <div class="salutation">Bonjour {{ user_display_name }},</div>
+                                <div class="summarytext">{{ summary_text }}</div>
+                            </td>
+                            <td class="logo">
+                                {% if app_name == "Riot" %}
+                                    <img src="http://matrix.org/img/riot-logo-email.png" width="83" height="83" alt="[Riot]"/>
+                                {% elif app_name == "Vector" %}
+                                    <img src="http://matrix.org/img/vector-logo-email.png" width="64" height="83" alt="[Vector]"/>
+                                {% else %}
+                                    <img src="http://matrix.org/img/matrix-120x51.png" width="120" height="51" alt="[matrix]"/>
+                                {% endif %}
+                            </td>
+                        </tr>
+                    </table>
+                    {% for room in rooms %}
+                        {% include 'room.html' with context %}
+                    {% endfor %}
+                    <div class="footer">
+                        <a href="{{ unsubscribe_link }}">Se désinscrire</a>
+                        <br/>
+                        <br/>
+                        <div class="debug">
+                            Sending email at {{ reason.now|format_ts("%c") }} due to activity in room {{ reason.room_name }} because
+                            an event was received at {{ reason.received_at|format_ts("%c") }}
+                            which is more than {{ "%.1f"|format(reason.delay_before_mail_ms / (60*1000)) }} ({{ reason.delay_before_mail_ms }}) mins ago,
+                            {% if reason.last_sent_ts %}
+                                and the last time we sent a mail for this room was {{ reason.last_sent_ts|format_ts("%c") }},
+                                which is more than {{ "%.1f"|format(reason.throttle_ms / (60*1000)) }} (current throttle_ms) mins ago.
+                            {% else %}
+                                and we don't have a last time we sent a mail for this room.
+                            {% endif %}
+                        </div>
+                    </div>
+                </td>
+                <td> </td>
+            </tr>
+        </table>
+    </body>
+</html>

res/templates-dinsic/notif_mail.txt

@@ -0,0 +1,10 @@
+Bonjour {{ user_display_name }},
+
+{{ summary_text }}
+
+{% for room in rooms %}
+{% include 'room.txt' with context %}
+{% endfor %}
+
+Vous pouvez désactiver ces notifications en cliquant ici {{ unsubscribe_link }}
+

res/templates-dinsic/room.html

@@ -0,0 +1,33 @@
+<table class="room">
+    <tr class="room_header">
+        <td class="room_avatar">
+            {% if room.avatar_url %}
+                <img alt="" src="{{ room.avatar_url|mxc_to_http(48,48) }}" />
+            {% else %}
+                {% if room.hash % 3 == 0 %}
+                    <img alt="" src="https://vector.im/beta/img/76cfa6.png"  />
+                {% elif room.hash % 3 == 1 %}
+                    <img alt="" src="https://vector.im/beta/img/50e2c2.png"  />
+                {% else %}
+                    <img alt="" src="https://vector.im/beta/img/f4c371.png"  />
+                {% endif %}
+            {% endif %}
+        </td>
+        <td class="room_name" colspan="2">
+            {{ room.title }}
+        </td>
+    </tr>
+    {% if room.invite %}
+        <tr>
+            <td></td>
+            <td>
+                <a href="{{ room.link }}">Rejoindre la conversation.</a>
+            </td>
+            <td></td>
+        </tr>
+    {% else %}
+        {% for notif in room.notifs %}
+            {% include 'notif.html' with context %}
+        {% endfor %}
+    {% endif %}
+</table>

res/templates-dinsic/room.txt

@@ -0,0 +1,9 @@
+{{ room.title }}
+
+{% if room.invite %}
+    Vous avez été invité, rejoignez la conversation en cliquant sur le lien suivant {{ room.link }}
+{% else %}
+    {% for notif in room.notifs %}
+        {% include 'notif.txt' with context %}
+    {% endfor %}
+{% endif %}

synapse/api/auth.py

@@ -207,6 +207,7 @@ class Auth(object):
             )
 
             user_id, app_service = yield self._get_appservice_user_id(request)
+
             if user_id:
                 request.authenticated_entity = user_id
 
@@ -268,39 +269,40 @@ class Auth(object):
                 errcode=Codes.MISSING_TOKEN
             )
 
-    @defer.inlineCallbacks
     def _get_appservice_user_id(self, request):
         app_service = self.store.get_app_service_by_token(
             self.get_access_token_from_request(
                 request, self.TOKEN_NOT_FOUND_HTTP_STATUS
             )
         )
+
         if app_service is None:
-            defer.returnValue((None, None))
+            return(None, None)
 
         if app_service.ip_range_whitelist:
             ip_address = IPAddress(self.hs.get_ip_from_request(request))
             if ip_address not in app_service.ip_range_whitelist:
-                defer.returnValue((None, None))
+                return(None, None)
 
         if b"user_id" not in request.args:
-            defer.returnValue((app_service.sender, app_service))
+            return(app_service.sender, app_service)
 
         user_id = request.args[b"user_id"][0].decode('utf8')
         if app_service.sender == user_id:
-            defer.returnValue((app_service.sender, app_service))
+            return(app_service.sender, app_service)
 
         if not app_service.is_interested_in_user(user_id):
             raise AuthError(
                 403,
                 "Application service cannot masquerade as this user."
             )
-        if not (yield self.store.get_user_by_id(user_id)):
-            raise AuthError(
-                403,
-                "Application service has not registered this user"
-            )
-        defer.returnValue((user_id, app_service))
+        # Let ASes manipulate nonexistent users (e.g. to shadow-register them)
+        # if not (yield self.store.get_user_by_id(user_id)):
+        #     raise AuthError(
+        #         403,
+        #         "Application service has not registered this user"
+        #     )
+        return(user_id, app_service)
 
     @defer.inlineCallbacks
     def get_user_by_access_token(self, token, rights="access"):
@@ -533,24 +535,15 @@ class Auth(object):
         defer.returnValue(user_info)
 
     def get_appservice_by_req(self, request):
-        try:
-            token = self.get_access_token_from_request(
-                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
-            )
-            service = self.store.get_app_service_by_token(token)
-            if not service:
-                logger.warn("Unrecognised appservice access token.")
-                raise AuthError(
-                    self.TOKEN_NOT_FOUND_HTTP_STATUS,
-                    "Unrecognised access token.",
-                    errcode=Codes.UNKNOWN_TOKEN
-                )
-            request.authenticated_entity = service.sender
-            return defer.succeed(service)
-        except KeyError:
+        (user_id, app_service) = self._get_appservice_user_id(request)
+        if not app_service:
             raise AuthError(
-                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token."
+                self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                "Unrecognised access token.",
+                errcode=Codes.UNKNOWN_TOKEN,
             )
+        request.authenticated_entity = app_service.sender
+        return app_service
 
     def is_server_admin(self, user):
         """ Check if the given user is a local server admin.

synapse/api/constants.py

@@ -79,6 +79,7 @@ class EventTypes(object):
     RoomAvatar = "m.room.avatar"
     RoomEncryption = "m.room.encryption"
     GuestAccess = "m.room.guest_access"
+    Encryption = "m.room.encryption"
 
     # These are used for validation
     Message = "m.room.message"

synapse/api/errors.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,6 +62,13 @@ class Codes(object):
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
     WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
     EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
+    PASSWORD_TOO_SHORT = "M_PASSWORD_TOO_SHORT"
+    PASSWORD_NO_DIGIT = "M_PASSWORD_NO_DIGIT"
+    PASSWORD_NO_UPPERCASE = "M_PASSWORD_NO_UPPERCASE"
+    PASSWORD_NO_LOWERCASE = "M_PASSWORD_NO_LOWERCASE"
+    PASSWORD_NO_SYMBOL = "M_PASSWORD_NO_SYMBOL"
+    PASSWORD_IN_DICTIONARY = "M_PASSWORD_IN_DICTIONARY"
+    WEAK_PASSWORD = "M_WEAK_PASSWORD"
 
 
 class CodeMessageException(RuntimeError):
@@ -349,6 +357,22 @@ class IncompatibleRoomVersionError(SynapseError):
         )
 
 
+class PasswordRefusedError(SynapseError):
+    """A password has been refused, either during password reset/change or registration.
+    """
+
+    def __init__(
+        self,
+        msg="This password doesn't comply with the server's policy",
+        errcode=Codes.WEAK_PASSWORD,
+    ):
+        super(PasswordRefusedError, self).__init__(
+            code=400,
+            msg=msg,
+            errcode=errcode,
+        )
+
+
 class RequestSendFailed(RuntimeError):
     """Sending a HTTP request over federation failed due to not being able to
     talk to the remote server for some reason.

synapse/appservice/__init__.py

@@ -265,7 +265,7 @@ class ApplicationService(object):
     def is_exclusive_room(self, room_id):
         return self._is_exclusive(ApplicationService.NS_ROOMS, room_id)
 
-    def get_exlusive_user_regexes(self):
+    def get_exclusive_user_regexes(self):
         """Get the list of regexes used to determine if a user is exclusively
         registered by the AS
         """

synapse/config/password.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2015-2016 OpenMarket Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,6 +30,10 @@ class PasswordConfig(Config):
         self.password_enabled = password_config.get("enabled", True)
         self.password_pepper = password_config.get("pepper", "")
 
+        # Password policy
+        self.password_policy = password_config.get("policy", {})
+        self.password_policy_enabled = self.password_policy.pop("enabled", False)
+
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """\
         password_config:
@@ -39,4 +45,34 @@ class PasswordConfig(Config):
            # DO NOT CHANGE THIS AFTER INITIAL SETUP!
            #
            #pepper: "EVEN_MORE_SECRET"
+
+           # Define and enforce a password policy. Each parameter is optional, boolean
+           # parameters default to 'false' and integer parameters default to 0.
+           # This is an early implementation of MSC2000.
+           #
+           #policy:
+              # Whether to enforce the password policy.
+              #
+              #enabled: true
+
+              # Minimum accepted length for a password.
+              #
+              #minimum_length: 15
+
+              # Whether a password must contain at least one digit.
+              #
+              #require_digit: true
+
+              # Whether a password must contain at least one symbol.
+              # A symbol is any character that's not a number or a letter.
+              #
+              #require_symbol: true
+
+              # Whether a password must contain at least one lowercase letter.
+              #
+              #require_lowercase: true
+
+              # Whether a password must contain at least one lowercase letter.
+              #
+              #require_uppercase: true
         """

synapse/config/registration.py

@@ -39,6 +39,8 @@ class AccountValidityConfig(Config):
             else:
                 self.renew_email_subject = "Renew your %(app)s account"
 
+            self.startup_job_max_delta = self.period * 10. / 100.
+
         if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
             raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
@@ -60,8 +62,19 @@ class RegistrationConfig(Config):
 
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
         self.allowed_local_3pids = config.get("allowed_local_3pids", [])
+        self.check_is_for_allowed_local_3pids = config.get(
+            "check_is_for_allowed_local_3pids", None
+        )
+        self.allow_invited_3pids = config.get("allow_invited_3pids", False)
+
+        self.disable_3pid_changes = config.get("disable_3pid_changes", False)
+
         self.enable_3pid_lookup = config.get("enable_3pid_lookup", True)
         self.registration_shared_secret = config.get("registration_shared_secret")
+        self.register_mxid_from_3pid = config.get("register_mxid_from_3pid")
+        self.register_just_use_email_for_display_name = config.get(
+            "register_just_use_email_for_display_name", False,
+        )
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
         self.trusted_third_party_id_servers = config.get(
@@ -81,6 +94,16 @@ class RegistrationConfig(Config):
                 raise ConfigError('Invalid auto_join_rooms entry %s' % (room_alias,))
         self.autocreate_auto_join_rooms = config.get("autocreate_auto_join_rooms", True)
 
+        self.disable_set_displayname = config.get("disable_set_displayname", False)
+        self.disable_set_avatar_url = config.get("disable_set_avatar_url", False)
+
+        self.replicate_user_profiles_to = config.get("replicate_user_profiles_to", [])
+        if not isinstance(self.replicate_user_profiles_to, list):
+            self.replicate_user_profiles_to = [self.replicate_user_profiles_to, ]
+
+        self.shadow_server = config.get("shadow_server", None)
+        self.rewrite_identity_server_urls = config.get("rewrite_identity_server_urls", {})
+
         self.disable_msisdn_registration = (
             config.get("disable_msisdn_registration", False)
         )
@@ -129,7 +152,9 @@ class RegistrationConfig(Config):
         # This means that, if a validity period is set, and Synapse is restarted (it will
         # then derive an expiration date from the current validity period), and some time
         # after that the validity period changes and Synapse is restarted, the users'
-        # expiration dates won't be updated unless their account is manually renewed.
+        # expiration dates won't be updated unless their account is manually renewed. This
+        # date will be randomly selected within a range [now + period - d ; now + period],
+        # where d is equal to 10%% of the validity period.
         #
         #account_validity:
         #  enabled: True
@@ -148,9 +173,32 @@ class RegistrationConfig(Config):
         #
         #disable_msisdn_registration: true
 
+        # Derive the user's matrix ID from a type of 3PID used when registering.
+        # This overrides any matrix ID the user proposes when calling /register
+        # The 3PID type should be present in registrations_require_3pid to avoid
+        # users failing to register if they don't specify the right kind of 3pid.
+        #
+        #register_mxid_from_3pid: email
+
+        # Uncomment to set the display name of new users to their email address,
+        # rather than using the default heuristic.
+        #
+        #register_just_use_email_for_display_name: true
+
         # Mandate that users are only allowed to associate certain formats of
         # 3PIDs with accounts on this server.
         #
+        # Use an Identity Server to establish which 3PIDs are allowed to register?
+        # Overrides allowed_local_3pids below.
+        #
+        #check_is_for_allowed_local_3pids: matrix.org
+        #
+        # If you are using an IS you can also check whether that IS registers
+        # pending invites for the given 3PID (and then allow it to sign up on
+        # the platform):
+        #
+        #allow_invited_3pids: False
+        #
         #allowed_local_3pids:
         #  - medium: email
         #    pattern: '.*@matrix\\.org'
@@ -159,6 +207,11 @@ class RegistrationConfig(Config):
         #  - medium: msisdn
         #    pattern: '\\+44'
 
+        # If true, stop users from trying to change the 3PIDs associated with
+        # their accounts.
+        #
+        #disable_3pid_changes: False
+
         # Enable 3PIDs lookup requests to identity servers from this server.
         #
         #enable_3pid_lookup: true
@@ -200,6 +253,30 @@ class RegistrationConfig(Config):
         #  - matrix.org
         #  - vector.im
 
+        # If enabled, user IDs, display names and avatar URLs will be replicated
+        # to this server whenever they change.
+        # This is an experimental API currently implemented by sydent to support
+        # cross-homeserver user directories.
+        #
+        #replicate_user_profiles_to: example.com
+
+        # If specified, attempt to replay registrations, profile changes & 3pid
+        # bindings on the given target homeserver via the AS API. The HS is authed
+        # via a given AS token.
+        #
+        #shadow_server:
+        #  hs_url: https://shadow.example.com
+        #  hs: shadow.example.com
+        #  as_token: 12u394refgbdhivsia
+
+        # If enabled, don't let users set their own display names/avatars
+        # other than for the very first time (unless they are a server admin).
+        # Useful when provisioning users based on the contents of a 3rd party
+        # directory and to avoid ambiguities.
+        #
+        #disable_set_displayname: False
+        #disable_set_avatar_url: False
+
         # Users who register on this homeserver will automatically be joined
         # to these rooms
         #

synapse/config/user_directory.py

@@ -24,6 +24,7 @@ class UserDirectoryConfig(Config):
     def read_config(self, config):
         self.user_directory_search_enabled = True
         self.user_directory_search_all_users = False
+        self.user_directory_defer_to_id_server = None
         user_directory_config = config.get("user_directory", None)
         if user_directory_config:
             self.user_directory_search_enabled = (
@@ -32,6 +33,9 @@ class UserDirectoryConfig(Config):
             self.user_directory_search_all_users = (
                 user_directory_config.get("search_all_users", False)
             )
+            self.user_directory_defer_to_id_server = (
+                user_directory_config.get("defer_to_id_server", None)
+            )
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
@@ -50,4 +54,9 @@ class UserDirectoryConfig(Config):
         #user_directory:
         #  enabled: true
         #  search_all_users: false
+        #
+        #  # If this is set, user search will be delegated to this ID server instead
+        #  # of synapse performing the search itself.
+        #  # This is an experimental API.
+        #  defer_to_id_server: https://id.example.com
         """

synapse/events/spamcheck.py

@@ -46,13 +46,26 @@ class SpamChecker(object):
 
         return self.spam_checker.check_event_for_spam(event)
 
-    def user_may_invite(self, inviter_userid, invitee_userid, room_id):
+    def user_may_invite(self, inviter_userid, invitee_userid, third_party_invite,
+                        room_id, new_room, published_room):
         """Checks if a given user may send an invite
 
         If this method returns false, the invite will be rejected.
 
         Args:
-            userid (string): The sender's user ID
+            inviter_userid (str)
+            invitee_userid (str|None): The user ID of the invitee. Is None
+                if this is a third party invite and the 3PID is not bound to a
+                user ID.
+            third_party_invite (dict|None): If a third party invite then is a
+                dict containing the medium and address of the invitee.
+            room_id (str)
+            new_room (bool): Whether the user is being invited to the room as
+                part of a room creation, if so the invitee would have been
+                included in the call to `user_may_create_room`.
+            published_room (bool): Whether the room the user is being invited
+                to has been published in the local homeserver's public room
+                directory.
 
         Returns:
             bool: True if the user may send an invite, otherwise False
@@ -60,15 +73,25 @@ class SpamChecker(object):
         if self.spam_checker is None:
             return True
 
-        return self.spam_checker.user_may_invite(inviter_userid, invitee_userid, room_id)
+        return self.spam_checker.user_may_invite(
+            inviter_userid, invitee_userid, third_party_invite, room_id, new_room,
+            published_room,
+        )
 
-    def user_may_create_room(self, userid):
+    def user_may_create_room(self, userid, invite_list, third_party_invite_list,
+                             cloning):
         """Checks if a given user may create a room
 
         If this method returns false, the creation request will be rejected.
 
         Args:
             userid (string): The sender's user ID
+            invite_list (list[str]): List of user IDs that would be invited to
+                the new room.
+            third_party_invite_list (list[dict]): List of third party invites
+                for the new room.
+            cloning (bool): Whether the user is cloning an existing room, e.g.
+                upgrading a room.
 
         Returns:
             bool: True if the user may create a room, otherwise False
@@ -76,7 +99,9 @@ class SpamChecker(object):
         if self.spam_checker is None:
             return True
 
-        return self.spam_checker.user_may_create_room(userid)
+        return self.spam_checker.user_may_create_room(
+            userid, invite_list, third_party_invite_list, cloning,
+        )
 
     def user_may_create_room_alias(self, userid, room_alias):
         """Checks if a given user may create a room alias
@@ -111,3 +136,21 @@ class SpamChecker(object):
             return True
 
         return self.spam_checker.user_may_publish_room(userid, room_id)
+
+    def user_may_join_room(self, userid, room_id, is_invited):
+        """Checks if a given users is allowed to join a room.
+
+        Is not called when the user creates a room.
+
+        Args:
+            userid (str)
+            room_id (str)
+            is_invited (bool): Whether the user is invited into the room
+
+        Returns:
+            bool: Whether the user may join the room
+        """
+        if self.spam_checker is None:
+            return True
+
+        return self.spam_checker.user_may_join_room(userid, room_id, is_invited)

synapse/handlers/deactivate_account.py

@@ -33,6 +33,7 @@ class DeactivateAccountHandler(BaseHandler):
         self._device_handler = hs.get_device_handler()
         self._room_member_handler = hs.get_room_member_handler()
         self._identity_handler = hs.get_handlers().identity_handler
+        self._profile_handler = hs.get_profile_handler()
         self.user_directory_handler = hs.get_user_directory_handler()
 
         # Flag that indicates whether the process to part users from rooms is running
@@ -98,6 +99,9 @@ class DeactivateAccountHandler(BaseHandler):
 
         yield self.store.user_set_password_hash(user_id, None)
 
+        user = UserID.from_string(user_id)
+        yield self._profile_handler.set_active(user, False, False)
+
         # Add the user to a table of users pending deactivation (ie.
         # removal from all the rooms they're a member of)
         yield self.store.add_user_pending_deactivation(user_id)

synapse/handlers/device.py

@@ -568,6 +568,12 @@ class DeviceListEduUpdater(object):
                 stream_id = result["stream_id"]
                 devices = result["devices"]
 
+                for device in devices:
+                    logger.debug(
+                        "Handling resync update %r/%r, ID: %r",
+                        user_id, device["device_id"], stream_id,
+                    )
+
                 # If the remote server has more than ~1000 devices for this user
                 # we assume that something is going horribly wrong (e.g. a bot
                 # that logs in and creates a new device every time it tries to

synapse/handlers/federation.py

@@ -1340,8 +1340,12 @@ class FederationHandler(BaseHandler):
         if self.hs.config.block_non_admin_invites:
             raise SynapseError(403, "This server does not accept room invites")
 
+        is_published = yield self.store.is_room_published(event.room_id)
+
         if not self.spam_checker.user_may_invite(
-            event.sender, event.state_key, event.room_id,
+            event.sender, event.state_key, None,
+            room_id=event.room_id, new_room=False,
+            published_room=is_published,
         ):
             raise SynapseError(
                 403, "This user is not permitted to send invites to this server/user"

synapse/handlers/identity.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018, 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,13 +20,18 @@
 import logging
 
 from canonicaljson import json
+from signedjson.key import decode_verify_key_bytes
+from signedjson.sign import verify_signed_json
+from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
 from synapse.api.errors import (
+    AuthError,
     CodeMessageException,
     Codes,
     HttpResponseException,
+    ProxiedRequestError,
     SynapseError,
 )
 
@@ -47,6 +52,8 @@ class IdentityHandler(BaseHandler):
         self.trust_any_id_server_just_for_testing_do_not_use = (
             hs.config.use_insecure_ssl_client_just_for_testing_do_not_use
         )
+        self.rewrite_identity_server_urls = hs.config.rewrite_identity_server_urls
+        self._enable_lookup = hs.config.enable_3pid_lookup
 
     def _should_trust_id_server(self, id_server):
         if id_server not in self.trusted_id_servers:
@@ -84,7 +91,10 @@ class IdentityHandler(BaseHandler):
                 'credentials', id_server
             )
             defer.returnValue(None)
-
+        # if we have a rewrite rule set for the identity server,
+        # apply it now.
+        if id_server in self.rewrite_identity_server_urls:
+            id_server = self.rewrite_identity_server_urls[id_server]
         try:
             data = yield self.http_client.get_json(
                 "https://%s%s" % (
@@ -119,7 +129,10 @@ class IdentityHandler(BaseHandler):
             client_secret = creds['clientSecret']
         else:
             raise SynapseError(400, "No client_secret in creds")
-
+        # if we have a rewrite rule set for the identity server,
+        # apply it now.
+        if id_server in self.rewrite_identity_server_urls:
+            id_server = self.rewrite_identity_server_urls[id_server]
         try:
             data = yield self.http_client.post_urlencoded_get_json(
                 "https://%s%s" % (
@@ -221,6 +234,16 @@ class IdentityHandler(BaseHandler):
             b"Authorization": auth_headers,
         }
 
+        # if we have a rewrite rule set for the identity server,
+        # apply it now.
+        #
+        # Note that destination_is has to be the real id_server, not
+        # the server we connect to.
+        if id_server in self.rewrite_identity_server_urls:
+            id_server = self.rewrite_identity_server_urls[id_server]
+
+        url = "https://%s/_matrix/identity/api/v1/3pid/unbind" % (id_server,)
+
         try:
             yield self.http_client.post_json_get_json(
                 url,
@@ -260,7 +283,10 @@ class IdentityHandler(BaseHandler):
             'send_attempt': send_attempt,
         }
         params.update(kwargs)
-
+        # if we have a rewrite rule set for the identity server,
+        # apply it now.
+        if id_server in self.rewrite_identity_server_urls:
+            id_server = self.rewrite_identity_server_urls[id_server]
         try:
             data = yield self.http_client.post_json_get_json(
                 "https://%s%s" % (
@@ -292,7 +318,10 @@ class IdentityHandler(BaseHandler):
             'send_attempt': send_attempt,
         }
         params.update(kwargs)
-
+        # if we have a rewrite rule set for the identity server,
+        # apply it now.
+        if id_server in self.rewrite_identity_server_urls:
+            id_server = self.rewrite_identity_server_urls[id_server]
         try:
             data = yield self.http_client.post_json_get_json(
                 "https://%s%s" % (
@@ -305,3 +334,125 @@ class IdentityHandler(BaseHandler):
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
+
+    @defer.inlineCallbacks
+    def lookup_3pid(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server.
+
+        Args:
+            id_server (str): The server name (including port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            Deferred[dict]: The result of the lookup. See
+            https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup
+            for details
+        """
+        if not self._should_trust_id_server(id_server):
+            raise SynapseError(
+                400, "Untrusted ID server '%s'" % id_server,
+                Codes.SERVER_NOT_TRUSTED
+            )
+
+        if not self._enable_lookup:
+            raise AuthError(
+                403, "Looking up third-party identifiers is denied from this server",
+            )
+
+        target = self.rewrite_identity_server_urls.get(id_server, id_server)
+
+        try:
+            data = yield self.http_client.get_json(
+                "https://%s/_matrix/identity/api/v1/lookup" % (target,),
+                {
+                    "medium": medium,
+                    "address": address,
+                }
+            )
+
+            if "mxid" in data:
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                yield self._verify_any_signature(data, id_server)
+
+        except HttpResponseException as e:
+            logger.info("Proxied lookup failed: %r", e)
+            raise e.to_synapse_error()
+        except IOError as e:
+            logger.info("Failed to contact %r: %s", id_server, e)
+            raise ProxiedRequestError(503, "Failed to contact identity server")
+
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def bulk_lookup_3pid(self, id_server, threepids):
+        """Looks up given 3pids in the passed identity server.
+
+        Args:
+            id_server (str): The server name (including port, if required)
+                of the identity server to use.
+            threepids ([[str, str]]): The third party identifiers to lookup, as
+                a list of 2-string sized lists ([medium, address]).
+
+        Returns:
+            Deferred[dict]: The result of the lookup. See
+            https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup
+            for details
+        """
+        if not self._should_trust_id_server(id_server):
+            raise SynapseError(
+                400, "Untrusted ID server '%s'" % id_server,
+                Codes.SERVER_NOT_TRUSTED
+            )
+
+        if not self._enable_lookup:
+            raise AuthError(
+                403, "Looking up third-party identifiers is denied from this server",
+            )
+
+        target = self.rewrite_identity_server_urls.get(id_server, id_server)
+
+        try:
+            data = yield self.http_client.post_json_get_json(
+                "https://%s/_matrix/identity/api/v1/bulk_lookup" % (target,),
+                {
+                    "threepids": threepids,
+                }
+            )
+
+        except HttpResponseException as e:
+            logger.info("Proxied lookup failed: %r", e)
+            raise e.to_synapse_error()
+        except IOError as e:
+            logger.info("Failed to contact %r: %s", id_server, e)
+            raise ProxiedRequestError(503, "Failed to contact identity server")
+
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def _verify_any_signature(self, data, server_hostname):
+        if server_hostname not in data["signatures"]:
+            raise AuthError(401, "No signature from server %s" % (server_hostname,))
+
+        for key_name, signature in data["signatures"][server_hostname].items():
+            target = self.rewrite_identity_server_urls.get(
+                server_hostname, server_hostname,
+            )
+
+            key_data = yield self.http_client.get_json(
+                "https://%s/_matrix/identity/api/v1/pubkey/%s" %
+                (target, key_name,),
+            )
+            if "public_key" not in key_data:
+                raise AuthError(401, "No public key named %s from %s" %
+                                (key_name, server_hostname,))
+            verify_signed_json(
+                data,
+                server_hostname,
+                decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
+            )
+            return
+
+        raise AuthError(401, "No signature from server %s" % (server_hostname,))

synapse/handlers/password_policy.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from synapse.api.errors import Codes, PasswordRefusedError
+
+logger = logging.getLogger(__name__)
+
+
+class PasswordPolicyHandler(object):
+    def __init__(self, hs):
+        self.policy = hs.config.password_policy
+        self.enabled = hs.config.password_policy_enabled
+
+        # Regexps for the spec'd policy parameters.
+        self.regexp_digit = re.compile("[0-9]")
+        self.regexp_symbol = re.compile("[^a-zA-Z0-9]")
+        self.regexp_uppercase = re.compile("[A-Z]")
+        self.regexp_lowercase = re.compile("[a-z]")
+
+    def validate_password(self, password):
+        """Checks whether a given password complies with the server's policy.
+
+        Args:
+            password (str): The password to check against the server's policy.
+
+        Raises:
+            PasswordRefusedError: The password doesn't comply with the server's policy.
+        """
+
+        if not self.enabled:
+            return
+
+        minimum_accepted_length = self.policy.get("minimum_length", 0)
+        if len(password) < minimum_accepted_length:
+            raise PasswordRefusedError(
+                msg=(
+                    "The password must be at least %d characters long"
+                    % minimum_accepted_length
+                ),
+                errcode=Codes.PASSWORD_TOO_SHORT,
+            )
+
+        if (
+            self.policy.get("require_digit", False) and
+            self.regexp_digit.search(password) is None
+        ):
+            raise PasswordRefusedError(
+                msg="The password must include at least one digit",
+                errcode=Codes.PASSWORD_NO_DIGIT,
+            )
+
+        if (
+            self.policy.get("require_symbol", False) and
+            self.regexp_symbol.search(password) is None
+        ):
+            raise PasswordRefusedError(
+                msg="The password must include at least one symbol",
+                errcode=Codes.PASSWORD_NO_SYMBOL,
+            )
+
+        if (
+            self.policy.get("require_uppercase", False) and
+            self.regexp_uppercase.search(password) is None
+        ):
+            raise PasswordRefusedError(
+                msg="The password must include at least one uppercase letter",
+                errcode=Codes.PASSWORD_NO_UPPERCASE,
+            )
+
+        if (
+            self.policy.get("require_lowercase", False) and
+            self.regexp_lowercase.search(password) is None
+        ):
+            raise PasswordRefusedError(
+                msg="The password must include at least one lowercase letter",
+                errcode=Codes.PASSWORD_NO_LOWERCASE,
+            )

synapse/handlers/profile.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,11 @@
 
 import logging
 
-from twisted.internet import defer
+from six.moves import range
+
+from signedjson.sign import sign_json
+
+from twisted.internet import defer, reactor
 
 from synapse.api.errors import (
     AuthError,
@@ -26,6 +31,7 @@ from synapse.api.errors import (
 )
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import UserID, get_domain_from_id
+from synapse.util.logcontext import run_in_background
 
 from ._base import BaseHandler
 
@@ -40,6 +46,8 @@ class BaseProfileHandler(BaseHandler):
     subclass MasterProfileHandler
     """
 
+    PROFILE_REPLICATE_INTERVAL = 2 * 60 * 1000
+
     def __init__(self, hs):
         super(BaseProfileHandler, self).__init__(hs)
 
@@ -50,6 +58,84 @@ class BaseProfileHandler(BaseHandler):
 
         self.user_directory_handler = hs.get_user_directory_handler()
 
+        self.http_client = hs.get_simple_http_client()
+
+        if hs.config.worker_app is None:
+            self.clock.looping_call(
+                self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
+            )
+
+            if len(self.hs.config.replicate_user_profiles_to) > 0:
+                reactor.callWhenRunning(self._assign_profile_replication_batches)
+                reactor.callWhenRunning(self._replicate_profiles)
+                # Add a looping call to replicate_profiles: this handles retries
+                # if the replication is unsuccessful when the user updated their
+                # profile.
+                self.clock.looping_call(
+                    self._replicate_profiles, self.PROFILE_REPLICATE_INTERVAL
+                )
+
+    @defer.inlineCallbacks
+    def _assign_profile_replication_batches(self):
+        """If no profile replication has been done yet, allocate replication batch
+        numbers to each profile to start the replication process.
+        """
+        logger.info("Assigning profile batch numbers...")
+        total = 0
+        while True:
+            assigned = yield self.store.assign_profile_batch()
+            total += assigned
+            if assigned == 0:
+                break
+        logger.info("Assigned %d profile batch numbers", total)
+
+    @defer.inlineCallbacks
+    def _replicate_profiles(self):
+        """If any profile data has been updated and not pushed to the replication targets,
+        replicate it.
+        """
+        host_batches = yield self.store.get_replication_hosts()
+        latest_batch = yield self.store.get_latest_profile_replication_batch_number()
+        if latest_batch is None:
+            latest_batch = -1
+        for repl_host in self.hs.config.replicate_user_profiles_to:
+            if repl_host not in host_batches:
+                host_batches[repl_host] = -1
+            try:
+                for i in range(host_batches[repl_host] + 1, latest_batch + 1):
+                    yield self._replicate_host_profile_batch(repl_host, i)
+            except Exception:
+                logger.exception(
+                    "Exception while replicating to %s: aborting for now", repl_host,
+                )
+
+    @defer.inlineCallbacks
+    def _replicate_host_profile_batch(self, host, batchnum):
+        logger.info("Replicating profile batch %d to %s", batchnum, host)
+        batch_rows = yield self.store.get_profile_batch(batchnum)
+        batch = {
+            UserID(r["user_id"], self.hs.hostname).to_string(): ({
+                "display_name": r["displayname"],
+                "avatar_url": r["avatar_url"],
+            } if r["active"] else None) for r in batch_rows
+        }
+
+        url = "https://%s/_matrix/identity/api/v1/replicate_profiles" % (host,)
+        body = {
+            "batchnum": batchnum,
+            "batch": batch,
+            "origin_server": self.hs.hostname,
+        }
+        signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0])
+        try:
+            yield self.http_client.post_json_get_json(url, signed_body)
+            yield self.store.update_replication_batch_for_host(host, batchnum)
+            logger.info("Sucessfully replicated profile batch %d to %s", batchnum, host)
+        except Exception:
+            # This will get retried when the looping call next comes around
+            logger.exception("Failed to replicate profile batch %d to %s", batchnum, host)
+            raise
+
     @defer.inlineCallbacks
     def get_profile(self, user_id):
         target_user = UserID.from_string(user_id)
@@ -159,14 +245,25 @@ class BaseProfileHandler(BaseHandler):
         if not self.hs.is_mine(target_user):
             raise SynapseError(400, "User is not hosted on this Home Server")
 
-        if not by_admin and target_user != requester.user:
+        if not by_admin and requester and target_user != requester.user:
             raise AuthError(400, "Cannot set another user's displayname")
 
+        if not by_admin and self.hs.config.disable_set_displayname:
+            profile = yield self.store.get_profileinfo(target_user.localpart)
+            if profile.display_name:
+                raise SynapseError(400, "Changing displayname is disabled on this server")
+
         if new_displayname == '':
             new_displayname = None
 
+        if len(self.hs.config.replicate_user_profiles_to) > 0:
+            cur_batchnum = yield self.store.get_latest_profile_replication_batch_number()
+            new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1
+        else:
+            new_batchnum = None
+
         yield self.store.set_profile_displayname(
-            target_user.localpart, new_displayname
+            target_user.localpart, new_displayname, new_batchnum
         )
 
         if self.hs.config.user_directory_search_all_users:
@@ -175,7 +272,37 @@ class BaseProfileHandler(BaseHandler):
                 target_user.to_string(), profile
             )
 
-        yield self._update_join_states(requester, target_user)
+        if requester:
+            yield self._update_join_states(requester, target_user)
+
+        # start a profile replication push
+        run_in_background(self._replicate_profiles)
+
+    @defer.inlineCallbacks
+    def set_active(self, target_user, active, hide):
+        """
+        Sets the 'active' flag on a user profile. If set to false, the user
+        account is considered deactivated or hidden.
+
+        If 'hide' is true, then we interpret active=False as a request to try to
+        hide the user rather than deactivating it.  This means withholding the
+        profile from replication (and mark it as inactive) rather than clearing
+        the profile from the HS DB. Note that unlike set_displayname and
+        set_avatar_url, this does *not* perform authorization checks! This is
+        because the only place it's used currently is in account deactivation
+        where we've already done these checks anyway.
+        """
+        if len(self.hs.config.replicate_user_profiles_to) > 0:
+            cur_batchnum = yield self.store.get_latest_profile_replication_batch_number()
+            new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1
+        else:
+            new_batchnum = None
+        yield self.store.set_profile_active(
+            target_user.localpart, active, hide, new_batchnum
+        )
+
+        # start a profile replication push
+        run_in_background(self._replicate_profiles)
 
     @defer.inlineCallbacks
     def get_avatar_url(self, target_user):
@@ -217,8 +344,19 @@ class BaseProfileHandler(BaseHandler):
         if not by_admin and target_user != requester.user:
             raise AuthError(400, "Cannot set another user's avatar_url")
 
+        if not by_admin and self.hs.config.disable_set_avatar_url:
+            profile = yield self.store.get_profileinfo(target_user.localpart)
+            if profile.avatar_url:
+                raise SynapseError(400, "Changing avatar url is disabled on this server")
+
+        if len(self.hs.config.replicate_user_profiles_to) > 0:
+            cur_batchnum = yield self.store.get_latest_profile_replication_batch_number()
+            new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1
+        else:
+            new_batchnum = None
+
         yield self.store.set_profile_avatar_url(
-            target_user.localpart, new_avatar_url
+            target_user.localpart, new_avatar_url, new_batchnum,
         )
 
         if self.hs.config.user_directory_search_all_users:
@@ -229,6 +367,9 @@ class BaseProfileHandler(BaseHandler):
 
         yield self._update_join_states(requester, target_user)
 
+        # start a profile replication push
+        run_in_background(self._replicate_profiles)
+
     @defer.inlineCallbacks
     def on_profile_query(self, args):
         user = UserID.from_string(args["user_id"])

synapse/handlers/register.py

@@ -61,6 +61,7 @@ class RegistrationHandler(BaseHandler):
         self.profile_handler = hs.get_profile_handler()
         self.user_directory_handler = hs.get_user_directory_handler()
         self.captcha_client = CaptchaServerHttpClient(hs)
+        self.http_client = hs.get_simple_http_client()
         self.identity_handler = self.hs.get_handlers().identity_handler
         self.ratelimiter = hs.get_registration_ratelimiter()
 
@@ -225,6 +226,11 @@ class RegistrationHandler(BaseHandler):
                 address=address,
             )
 
+            if default_display_name:
+                yield self.profile_handler.set_displayname(
+                    user, None, default_display_name, by_admin=True,
+                )
+
             if self.hs.config.user_directory_search_all_users:
                 profile = yield self.store.get_profileinfo(localpart)
                 yield self.user_directory_handler.handle_local_profile_change(
@@ -254,6 +260,11 @@ class RegistrationHandler(BaseHandler):
                         create_profile_with_displayname=default_display_name,
                         address=address,
                     )
+
+                    yield self.profile_handler.set_displayname(
+                        user, None, default_display_name, by_admin=True,
+                    )
+
                 except SynapseError:
                     # if user id is taken, just generate another
                     user = None
@@ -347,7 +358,9 @@ class RegistrationHandler(BaseHandler):
         yield self._auto_join_rooms(user_id)
 
     @defer.inlineCallbacks
-    def appservice_register(self, user_localpart, as_token):
+    def appservice_register(self, user_localpart, as_token, password, display_name):
+        # FIXME: this should be factored out and merged with normal register()
+
         user = UserID(user_localpart, self.hs.hostname)
         user_id = user.to_string()
         service = self.store.get_app_service_by_token(as_token)
@@ -365,12 +378,29 @@ class RegistrationHandler(BaseHandler):
             user_id, allowed_appservice=service
         )
 
+        password_hash = ""
+        if password:
+            password_hash = yield self.auth_handler().hash(password)
+
+        display_name = display_name or user.localpart
+
         yield self.register_with_store(
             user_id=user_id,
-            password_hash="",
+            password_hash=password_hash,
             appservice_id=service_id,
-            create_profile_with_displayname=user.localpart,
+            create_profile_with_displayname=display_name,
         )
+
+        yield self.profile_handler.set_displayname(
+            user, None, display_name, by_admin=True,
+        )
+
+        if self.hs.config.user_directory_search_all_users:
+            profile = yield self.store.get_profileinfo(user_localpart)
+            yield self.user_directory_handler.handle_local_profile_change(
+                user_id, profile
+            )
+
         defer.returnValue(user_id)
 
     @defer.inlineCallbacks
@@ -396,6 +426,39 @@ class RegistrationHandler(BaseHandler):
         else:
             logger.info("Valid captcha entered from %s", ip)
 
+    @defer.inlineCallbacks
+    def register_saml2(self, localpart):
+        """
+        Registers email_id as SAML2 Based Auth.
+        """
+        if types.contains_invalid_mxid_characters(localpart):
+            raise SynapseError(
+                400,
+                "User ID can only contain characters a-z, 0-9, or '=_-./'",
+            )
+        yield self.auth.check_auth_blocking()
+        user = UserID(localpart, self.hs.hostname)
+        user_id = user.to_string()
+
+        yield self.check_user_id_not_appservice_exclusive(user_id)
+        token = self.macaroon_gen.generate_access_token(user_id)
+        try:
+            yield self.register_with_store(
+                user_id=user_id,
+                token=token,
+                password_hash=None,
+                create_profile_with_displayname=user.localpart,
+            )
+
+            yield self.profile_handler.set_displayname(
+                user, None, user.localpart, by_admin=True,
+            )
+        except Exception as e:
+            yield self.store.add_access_token_to_user(user_id, token)
+            # Ignore Registration errors
+            logger.exception(e)
+        defer.returnValue((user_id, token))
+
     @defer.inlineCallbacks
     def register_email(self, threepidCreds):
         """
@@ -418,7 +481,9 @@ class RegistrationHandler(BaseHandler):
             logger.info("got threepid with medium '%s' and address '%s'",
                         threepid['medium'], threepid['address'])
 
-            if not check_3pid_allowed(self.hs, threepid['medium'], threepid['address']):
+            if not (
+                yield check_3pid_allowed(self.hs, threepid['medium'], threepid['address'])
+            ):
                 raise RegistrationError(
                     403, "Third party identifier is not allowed"
                 )
@@ -459,6 +524,39 @@ class RegistrationHandler(BaseHandler):
                     errcode=Codes.EXCLUSIVE
                 )
 
+    @defer.inlineCallbacks
+    def shadow_register(self, localpart, display_name, auth_result, params):
+        """Invokes the current registration on another server, using
+        shared secret registration, passing in any auth_results from
+        other registration UI auth flows (e.g. validated 3pids)
+        Useful for setting up shadow/backup accounts on a parallel deployment.
+        """
+
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.post_json_get_json(
+            "%s/_matrix/client/r0/register?access_token=%s" % (
+                shadow_hs_url, as_token,
+            ),
+            {
+                # XXX: auth_result is an unspecified extension for shadow registration
+                'auth_result': auth_result,
+                # XXX: another unspecified extension for shadow registration to ensure
+                # that the displayname is correctly set by the masters erver
+                'display_name': display_name,
+                'username': localpart,
+                'password': params.get("password"),
+                'bind_email': params.get("bind_email"),
+                'bind_msisdn': params.get("bind_msisdn"),
+                'device_id': params.get("device_id"),
+                'initial_device_display_name': params.get("initial_device_display_name"),
+                'inhibit_login': False,
+                'access_token': as_token,
+            }
+        )
+
     @defer.inlineCallbacks
     def _generate_user_id(self, reseed=False):
         if reseed or self._next_generated_user_id is None:
@@ -545,18 +643,16 @@ class RegistrationHandler(BaseHandler):
                 user_id=user_id,
                 token=token,
                 password_hash=password_hash,
-                create_profile_with_displayname=user.localpart,
+                create_profile_with_displayname=displayname or user.localpart,
             )
+            if displayname is not None:
+                yield self.profile_handler.set_displayname(
+                    user, None, displayname or user.localpart, by_admin=True,
+                )
         else:
             yield self._auth_handler.delete_access_tokens_for_user(user_id)
             yield self.store.add_access_token_to_user(user_id=user_id, token=token)
 
-        if displayname is not None:
-            logger.info("setting user display name: %s -> %s", user_id, displayname)
-            yield self.profile_handler.set_displayname(
-                user, requester, displayname, by_admin=True,
-            )
-
         defer.returnValue((user_id, token))
 
     @defer.inlineCallbacks

synapse/handlers/room.py

@@ -49,12 +49,14 @@ class RoomCreationHandler(BaseHandler):
             "history_visibility": "shared",
             "original_invitees_have_ops": False,
             "guest_can_join": True,
+            "encryption_alg": "m.megolm.v1.aes-sha2",
         },
         RoomCreationPreset.TRUSTED_PRIVATE_CHAT: {
             "join_rules": JoinRules.INVITE,
             "history_visibility": "shared",
             "original_invitees_have_ops": True,
             "guest_can_join": True,
+            "encryption_alg": "m.megolm.v1.aes-sha2",
         },
         RoomCreationPreset.PUBLIC_CHAT: {
             "join_rules": JoinRules.PUBLIC,
@@ -74,6 +76,8 @@ class RoomCreationHandler(BaseHandler):
         # linearizer to stop two upgrades happening at once
         self._upgrade_linearizer = Linearizer("room_upgrade_linearizer")
 
+        self._server_notices_mxid = hs.config.server_notices_mxid
+
     @defer.inlineCallbacks
     def upgrade_room(self, requester, old_room_id, new_version):
         """Replace a room with a new room with a different version
@@ -247,7 +251,22 @@ class RoomCreationHandler(BaseHandler):
         """
         user_id = requester.user.to_string()
 
-        if not self.spam_checker.user_may_create_room(user_id):
+        if (self._server_notices_mxid is not None and
+                requester.user.to_string() == self._server_notices_mxid):
+            # allow the server notices mxid to create rooms
+            is_requester_admin = True
+
+        else:
+            is_requester_admin = yield self.auth.is_server_admin(
+                requester.user,
+            )
+
+        if not is_requester_admin and not self.spam_checker.user_may_create_room(
+            user_id,
+            invite_list=[],
+            third_party_invite_list=[],
+            cloning=True,
+        ):
             raise SynapseError(403, "You are not permitted to create rooms")
 
         creation_content = {
@@ -469,7 +488,24 @@ class RoomCreationHandler(BaseHandler):
 
         yield self.auth.check_auth_blocking(user_id)
 
-        if not self.spam_checker.user_may_create_room(user_id):
+        invite_list = config.get("invite", [])
+        invite_3pid_list = config.get("invite_3pid", [])
+
+        if (self._server_notices_mxid is not None and
+                requester.user.to_string() == self._server_notices_mxid):
+            # allow the server notices mxid to create rooms
+            is_requester_admin = True
+        else:
+            is_requester_admin = yield self.auth.is_server_admin(
+                requester.user,
+            )
+
+        if not is_requester_admin and not self.spam_checker.user_may_create_room(
+            user_id,
+            invite_list=invite_list,
+            third_party_invite_list=invite_3pid_list,
+            cloning=False,
+        ):
             raise SynapseError(403, "You are not permitted to create rooms")
 
         if ratelimit:
@@ -512,7 +548,6 @@ class RoomCreationHandler(BaseHandler):
         else:
             room_alias = None
 
-        invite_list = config.get("invite", [])
         for i in invite_list:
             try:
                 UserID.from_string(i)
@@ -523,8 +558,6 @@ class RoomCreationHandler(BaseHandler):
             requester,
         )
 
-        invite_3pid_list = config.get("invite_3pid", [])
-
         visibility = config.get("visibility", None)
         is_public = visibility == "public"
 
@@ -610,6 +643,7 @@ class RoomCreationHandler(BaseHandler):
                 "invite",
                 ratelimit=False,
                 content=content,
+                new_room=True,
             )
 
         for invite_3pid in invite_3pid_list:
@@ -624,6 +658,7 @@ class RoomCreationHandler(BaseHandler):
                 id_server,
                 requester,
                 txn_id=None,
+                new_room=True,
             )
 
         result = {"room_id": room_id}
@@ -694,6 +729,7 @@ class RoomCreationHandler(BaseHandler):
             "join",
             ratelimit=False,
             content=creator_join_profile,
+            new_room=True,
         )
 
         # We treat the power levels override specially as this needs to be one
@@ -769,6 +805,15 @@ class RoomCreationHandler(BaseHandler):
                 content=content,
             )
 
+        if "encryption_alg" in config:
+            yield send(
+                etype=EventTypes.Encryption,
+                state_key="",
+                content={
+                    'algorithm': config["encryption_alg"],
+                }
+            )
+
     @defer.inlineCallbacks
     def _generate_room_id(self, creator_id, is_public):
         # autogen room IDs and try to create it. We may clash, so just

synapse/handlers/room_member.py

@@ -20,16 +20,12 @@ import logging
 
 from six.moves import http_client
 
-from signedjson.key import decode_verify_key_bytes
-from signedjson.sign import verify_signed_json
-from unpaddedbase64 import decode_base64
-
 from twisted.internet import defer
 
 import synapse.server
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
-from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.api.errors import AuthError, Codes, ProxiedRequestError, SynapseError
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -67,12 +63,14 @@ class RoomMemberHandler(object):
         self.registration_handler = hs.get_registration_handler()
         self.profile_handler = hs.get_profile_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
+        self.identity_handler = hs.get_handlers().identity_handler
 
         self.member_linearizer = Linearizer(name="member")
 
         self.clock = hs.get_clock()
         self.spam_checker = hs.get_spam_checker()
         self._server_notices_mxid = self.config.server_notices_mxid
+        self.rewrite_identity_server_urls = self.config.rewrite_identity_server_urls
         self._enable_lookup = hs.config.enable_3pid_lookup
         self.allow_per_room_profiles = self.config.allow_per_room_profiles
 
@@ -317,8 +315,31 @@ class RoomMemberHandler(object):
             third_party_signed=None,
             ratelimit=True,
             content=None,
+            new_room=False,
             require_consent=True,
     ):
+        """Update a users membership in a room
+
+        Args:
+            requester (Requester)
+            target (UserID)
+            room_id (str)
+            action (str): The "action" the requester is performing against the
+                target. One of join/leave/kick/ban/invite/unban.
+            txn_id (str|None): The transaction ID associated with the request,
+                or None not provided.
+            remote_room_hosts (list[str]|None): List of remote servers to try
+                and join via if server isn't already in the room.
+            third_party_signed (dict|None): The signed object for third party
+                invites.
+            ratelimit (bool): Whether to apply ratelimiting to this request.
+            content (dict|None): Fields to include in the new events content.
+            new_room (bool): Whether these membership changes are happening
+                as part of a room creation (e.g. initial joins and invites)
+
+        Returns:
+            Deferred[FrozenEvent]
+        """
         key = (room_id,)
 
         with (yield self.member_linearizer.queue(key)):
@@ -332,6 +353,7 @@ class RoomMemberHandler(object):
                 third_party_signed=third_party_signed,
                 ratelimit=ratelimit,
                 content=content,
+                new_room=new_room,
                 require_consent=require_consent,
             )
 
@@ -349,6 +371,7 @@ class RoomMemberHandler(object):
             third_party_signed=None,
             ratelimit=True,
             content=None,
+            new_room=False,
             require_consent=True,
     ):
         content_specified = bool(content)
@@ -416,8 +439,14 @@ class RoomMemberHandler(object):
                     )
                     block_invite = True
 
+                is_published = yield self.store.is_room_published(room_id)
+
                 if not self.spam_checker.user_may_invite(
-                    requester.user.to_string(), target.to_string(), room_id,
+                    requester.user.to_string(), target.to_string(),
+                    third_party_invite=None,
+                    room_id=room_id,
+                    new_room=new_room,
+                    published_room=is_published,
                 ):
                     logger.info("Blocking invite due to spam checker")
                     block_invite = True
@@ -496,8 +525,29 @@ class RoomMemberHandler(object):
                     # so don't really fit into the general auth process.
                     raise AuthError(403, "Guest access not allowed")
 
+            if (self._server_notices_mxid is not None and
+                    requester.user.to_string() == self._server_notices_mxid):
+                # allow the server notices mxid to join rooms
+                is_requester_admin = True
+
+            else:
+                is_requester_admin = yield self.auth.is_server_admin(
+                    requester.user,
+                )
+
+            inviter = yield self._get_inviter(target.to_string(), room_id)
+            if not is_requester_admin:
+                # We assume that if the spam checker allowed the user to create
+                # a room then they're allowed to join it.
+                if not new_room and not self.spam_checker.user_may_join_room(
+                    target.to_string(), room_id,
+                    is_invited=inviter is not None,
+                ):
+                    raise SynapseError(
+                        403, "Not allowed to join this room",
+                    )
+
             if not is_host_in_room:
-                inviter = yield self._get_inviter(target.to_string(), room_id)
                 if inviter and not self.hs.is_mine(inviter):
                     remote_room_hosts.append(inviter.domain)
 
@@ -707,7 +757,8 @@ class RoomMemberHandler(object):
             address,
             id_server,
             requester,
-            txn_id
+            txn_id,
+            new_room=False,
     ):
         if self.config.block_non_admin_invites:
             is_requester_admin = yield self.auth.is_server_admin(
@@ -727,6 +778,23 @@ class RoomMemberHandler(object):
             id_server, medium, address
         )
 
+        is_published = yield self.store.is_room_published(room_id)
+
+        if not self.spam_checker.user_may_invite(
+            requester.user.to_string(), invitee,
+            third_party_invite={
+                "medium": medium,
+                "address": address,
+            },
+            room_id=room_id,
+            new_room=new_room,
+            published_room=is_published,
+        ):
+            logger.info("Blocking invite due to spam checker")
+            raise SynapseError(
+                403, "Invites have been disabled on this server",
+            )
+
         if invitee:
             yield self.update_membership(
                 requester,
@@ -746,6 +814,20 @@ class RoomMemberHandler(object):
                 txn_id=txn_id
             )
 
+    def _get_id_server_target(self, id_server):
+        """Looks up an id_server's actual http endpoint
+
+        Args:
+            id_server (str): the server name to lookup.
+
+        Returns:
+            the http endpoint to connect to.
+        """
+        if id_server in self.rewrite_identity_server_urls:
+            return self.rewrite_identity_server_urls[id_server]
+
+        return id_server
+
     @defer.inlineCallbacks
     def _lookup_3pid(self, id_server, medium, address):
         """Looks up a 3pid in the passed identity server.
@@ -759,48 +841,13 @@ class RoomMemberHandler(object):
         Returns:
             str: the matrix ID of the 3pid, or None if it is not recognized.
         """
-        if not self._enable_lookup:
-            raise SynapseError(
-                403, "Looking up third-party identifiers is denied from this server",
-            )
         try:
-            data = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server,),
-                {
-                    "medium": medium,
-                    "address": address,
-                }
-            )
-
-            if "mxid" in data:
-                if "signatures" not in data:
-                    raise AuthError(401, "No signatures on 3pid binding")
-                yield self._verify_any_signature(data, id_server)
-                defer.returnValue(data["mxid"])
-
-        except IOError as e:
+            data = yield self.identity_handler.lookup_3pid(id_server, medium, address)
+            defer.returnValue(data.get("mxid"))
+        except ProxiedRequestError as e:
             logger.warn("Error from identity server lookup: %s" % (e,))
             defer.returnValue(None)
 
-    @defer.inlineCallbacks
-    def _verify_any_signature(self, data, server_hostname):
-        if server_hostname not in data["signatures"]:
-            raise AuthError(401, "No signature from server %s" % (server_hostname,))
-        for key_name, signature in data["signatures"][server_hostname].items():
-            key_data = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/api/v1/pubkey/%s" %
-                (id_server_scheme, server_hostname, key_name,),
-            )
-            if "public_key" not in key_data:
-                raise AuthError(401, "No public key named %s from %s" %
-                                (key_name, server_hostname,))
-            verify_signed_json(
-                data,
-                server_hostname,
-                decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
-            )
-            return
-
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(
             self,
@@ -926,8 +973,9 @@ class RoomMemberHandler(object):
                     user.
         """
 
+        target = self._get_id_server_target(id_server)
         is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
-            id_server_scheme, id_server,
+            id_server_scheme, target,
         )
 
         invite_config = {
@@ -967,7 +1015,7 @@ class RoomMemberHandler(object):
             fallback_public_key = {
                 "public_key": data["public_key"],
                 "key_validity_url": "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % (
-                    id_server_scheme, id_server,
+                    id_server_scheme, target,
                 ),
             }
         else:

synapse/handlers/set_password.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017 New Vector Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,9 +30,12 @@ class SetPasswordHandler(BaseHandler):
         super(SetPasswordHandler, self).__init__(hs)
         self._auth_handler = hs.get_auth_handler()
         self._device_handler = hs.get_device_handler()
+        self._password_policy_handler = hs.get_password_policy_handler()
 
     @defer.inlineCallbacks
     def set_password(self, user_id, newpassword, requester=None):
+        self._password_policy_handler.validate_password(newpassword)
+
         password_hash = yield self._auth_handler.hash(newpassword)
 
         except_device_id = requester.device_id if requester else None

synapse/rest/__init__.py

@@ -41,6 +41,7 @@ from synapse.rest.client.v2_alpha import (
     keys,
     notifications,
     openid,
+    password_policy,
     read_marker,
     receipts,
     register,
@@ -115,6 +116,7 @@ class ClientRestResource(JsonResource):
         room_upgrade_rest_servlet.register_servlets(hs, client_resource)
         capabilities.register_servlets(hs, client_resource)
         account_validity.register_servlets(hs, client_resource)
+        password_policy.register_servlets(hs, client_resource)
 
         # moving to /_synapse/admin
         synapse.rest.admin.register_servlets_for_client_rest_resource(

synapse/rest/client/v1/profile.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 """ This module contains REST servlets to do with profile: /profile/<paths> """
+import logging
+
 from twisted.internet import defer
 
 from synapse.http.servlet import parse_json_object_from_request
@@ -21,6 +23,8 @@ from synapse.types import UserID
 
 from .base import ClientV1RestServlet, client_path_patterns
 
+logger = logging.getLogger(__name__)
+
 
 class ProfileDisplaynameRestServlet(ClientV1RestServlet):
     PATTERNS = client_path_patterns("/profile/(?P<user_id>[^/]*)/displayname")
@@ -28,6 +32,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(ProfileDisplaynameRestServlet, self).__init__(hs)
         self.profile_handler = hs.get_profile_handler()
+        self.http_client = hs.get_simple_http_client()
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
@@ -65,11 +70,30 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
         yield self.profile_handler.set_displayname(
             user, requester, new_name, is_admin)
 
+        if self.hs.config.shadow_server:
+            shadow_user = UserID(
+                user.localpart, self.hs.config.shadow_server.get("hs")
+            )
+            self.shadow_displayname(shadow_user.to_string(), content)
+
         defer.returnValue((200, {}))
 
     def on_OPTIONS(self, request, user_id):
         return (200, {})
 
+    @defer.inlineCallbacks
+    def shadow_displayname(self, user_id, body):
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.put_json(
+            "%s/_matrix/client/r0/profile/%s/displayname?access_token=%s&user_id=%s" % (
+                shadow_hs_url, user_id, as_token, user_id
+            ),
+            body
+        )
+
 
 class ProfileAvatarURLRestServlet(ClientV1RestServlet):
     PATTERNS = client_path_patterns("/profile/(?P<user_id>[^/]*)/avatar_url")
@@ -77,6 +101,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(ProfileAvatarURLRestServlet, self).__init__(hs)
         self.profile_handler = hs.get_profile_handler()
+        self.http_client = hs.get_simple_http_client()
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
@@ -113,11 +138,30 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
         yield self.profile_handler.set_avatar_url(
             user, requester, new_name, is_admin)
 
+        if self.hs.config.shadow_server:
+            shadow_user = UserID(
+                user.localpart, self.hs.config.shadow_server.get("hs")
+            )
+            self.shadow_avatar_url(shadow_user.to_string(), content)
+
         defer.returnValue((200, {}))
 
     def on_OPTIONS(self, request, user_id):
         return (200, {})
 
+    @defer.inlineCallbacks
+    def shadow_avatar_url(self, user_id, body):
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.put_json(
+            "%s/_matrix/client/r0/profile/%s/avatar_url?access_token=%s&user_id=%s" % (
+                shadow_hs_url, user_id, as_token, user_id
+            ),
+            body
+        )
+
 
 class ProfileRestServlet(ClientV1RestServlet):
     PATTERNS = client_path_patterns("/profile/(?P<user_id>[^/]*)")

synapse/rest/client/v1/room.py

@@ -677,7 +677,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
                 content["address"],
                 content["id_server"],
                 requester,
-                txn_id
+                txn_id,
+                new_room=False,
             )
             defer.returnValue((200, {}))
             return

synapse/rest/client/v2_alpha/account.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018, 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import re
 
 from six.moves import http_client
 
@@ -26,7 +27,9 @@ from synapse.http.servlet import (
     RestServlet,
     assert_params_in_dict,
     parse_json_object_from_request,
+    parse_string,
 )
+from synapse.types import UserID
 from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.threepids import check_3pid_allowed
 
@@ -51,10 +54,10 @@ class EmailPasswordRequestTokenRestServlet(RestServlet):
             'id_server', 'client_secret', 'email', 'send_attempt'
         ])
 
-        if not check_3pid_allowed(self.hs, "email", body['email']):
+        if not (yield check_3pid_allowed(self.hs, "email", body['email'])):
             raise SynapseError(
                 403,
-                "Your email domain is not authorized on this server",
+                "Your email is not authorized on this server",
                 Codes.THREEPID_DENIED,
             )
 
@@ -89,7 +92,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
-        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+        if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)):
             raise SynapseError(
                 403,
                 "Account phone numbers are not authorized on this server",
@@ -117,6 +120,7 @@ class PasswordRestServlet(RestServlet):
         self.auth_handler = hs.get_auth_handler()
         self.datastore = self.hs.get_datastore()
         self._set_password_handler = hs.get_set_password_handler()
+        self.http_client = hs.get_simple_http_client()
 
     @interactive_auth_handler
     @defer.inlineCallbacks
@@ -135,9 +139,13 @@ class PasswordRestServlet(RestServlet):
 
         if self.auth.has_access_token(request):
             requester = yield self.auth.get_user_by_req(request)
-            params = yield self.auth_handler.validate_user_via_ui_auth(
-                requester, body, self.hs.get_ip_from_request(request),
-            )
+            # blindly trust ASes without UI-authing them
+            if requester.app_service:
+                params = body
+            else:
+                params = yield self.auth_handler.validate_user_via_ui_auth(
+                    requester, body, self.hs.get_ip_from_request(request),
+                )
             user_id = requester.user.to_string()
         else:
             requester = None
@@ -173,11 +181,30 @@ class PasswordRestServlet(RestServlet):
             user_id, new_password, requester
         )
 
+        if self.hs.config.shadow_server:
+            shadow_user = UserID(
+                requester.user.localpart, self.hs.config.shadow_server.get("hs")
+            )
+            self.shadow_password(params, shadow_user.to_string())
+
         defer.returnValue((200, {}))
 
     def on_OPTIONS(self, _):
         return 200, {}
 
+    @defer.inlineCallbacks
+    def shadow_password(self, body, user_id):
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.post_json_get_json(
+            "%s/_matrix/client/r0/account/password?access_token=%s&user_id=%s" % (
+                shadow_hs_url, as_token, user_id,
+            ),
+            body
+        )
+
 
 class DeactivateAccountRestServlet(RestServlet):
     PATTERNS = client_v2_patterns("/account/deactivate$")
@@ -244,10 +271,10 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
             ['id_server', 'client_secret', 'email', 'send_attempt'],
         )
 
-        if not check_3pid_allowed(self.hs, "email", body['email']):
+        if not (yield check_3pid_allowed(self.hs, "email", body['email'])):
             raise SynapseError(
                 403,
-                "Your email domain is not authorized on this server",
+                "Your email is not authorized on this server",
                 Codes.THREEPID_DENIED,
             )
 
@@ -281,7 +308,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
-        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+        if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)):
             raise SynapseError(
                 403,
                 "Account phone numbers are not authorized on this server",
@@ -308,7 +335,8 @@ class ThreepidRestServlet(RestServlet):
         self.identity_handler = hs.get_handlers().identity_handler
         self.auth = hs.get_auth()
         self.auth_handler = hs.get_auth_handler()
-        self.datastore = self.hs.get_datastore()
+        self.datastore = hs.get_datastore()
+        self.http_client = hs.get_simple_http_client()
 
     @defer.inlineCallbacks
     def on_GET(self, request):
@@ -322,27 +350,38 @@ class ThreepidRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        body = parse_json_object_from_request(request)
+        if self.hs.config.disable_3pid_changes:
+            raise SynapseError(400, "3PID changes disabled on this server")
 
-        threePidCreds = body.get('threePidCreds')
-        threePidCreds = body.get('three_pid_creds', threePidCreds)
-        if threePidCreds is None:
-            raise SynapseError(400, "Missing param", Codes.MISSING_PARAM)
+        body = parse_json_object_from_request(request)
 
         requester = yield self.auth.get_user_by_req(request)
         user_id = requester.user.to_string()
 
-        threepid = yield self.identity_handler.threepid_from_creds(threePidCreds)
+        # skip validation if this is a shadow 3PID from an AS
+        if not requester.app_service:
+            threePidCreds = body.get('threePidCreds')
+            threePidCreds = body.get('three_pid_creds', threePidCreds)
+            if threePidCreds is None:
+                raise SynapseError(400, "Missing param", Codes.MISSING_PARAM)
 
-        if not threepid:
-            raise SynapseError(
-                400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED
-            )
+            threepid = yield self.identity_handler.threepid_from_creds(threePidCreds)
 
-        for reqd in ['medium', 'address', 'validated_at']:
-            if reqd not in threepid:
-                logger.warn("Couldn't add 3pid: invalid response from ID server")
-                raise SynapseError(500, "Invalid response from ID Server")
+            if not threepid:
+                raise SynapseError(
+                    400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED
+                )
+
+            for reqd in ['medium', 'address', 'validated_at']:
+                if reqd not in threepid:
+                    logger.warn("Couldn't add 3pid: invalid response from ID server")
+                    raise SynapseError(500, "Invalid response from ID Server")
+        else:
+            # XXX: ASes pass in a validated threepid directly to bypass the IS.
+            # This makes the API entirely change shape when we have an AS token;
+            # it really should be an entirely separate API - perhaps
+            # /account/3pid/replicate or something.
+            threepid = body.get('threepid')
 
         yield self.auth_handler.add_threepid(
             user_id,
@@ -351,7 +390,7 @@ class ThreepidRestServlet(RestServlet):
             threepid['validated_at'],
         )
 
-        if 'bind' in body and body['bind']:
+        if not requester.app_service and ('bind' in body and body['bind']):
             logger.debug(
                 "Binding threepid %s to %s",
                 threepid, user_id
@@ -360,19 +399,43 @@ class ThreepidRestServlet(RestServlet):
                 threePidCreds, user_id
             )
 
+        if self.hs.config.shadow_server:
+            shadow_user = UserID(
+                requester.user.localpart, self.hs.config.shadow_server.get("hs")
+            )
+            self.shadow_3pid({'threepid': threepid}, shadow_user.to_string())
+
         defer.returnValue((200, {}))
 
+    @defer.inlineCallbacks
+    def shadow_3pid(self, body, user_id):
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.post_json_get_json(
+            "%s/_matrix/client/r0/account/3pid?access_token=%s&user_id=%s" % (
+                shadow_hs_url, as_token, user_id,
+            ),
+            body
+        )
+
 
 class ThreepidDeleteRestServlet(RestServlet):
     PATTERNS = client_v2_patterns("/account/3pid/delete$")
 
     def __init__(self, hs):
         super(ThreepidDeleteRestServlet, self).__init__()
+        self.hs = hs
         self.auth = hs.get_auth()
         self.auth_handler = hs.get_auth_handler()
+        self.http_client = hs.get_simple_http_client()
 
     @defer.inlineCallbacks
     def on_POST(self, request):
+        if self.hs.config.disable_3pid_changes:
+            raise SynapseError(400, "3PID changes disabled on this server")
+
         body = parse_json_object_from_request(request)
         assert_params_in_dict(body, ['medium', 'address'])
 
@@ -390,6 +453,12 @@ class ThreepidDeleteRestServlet(RestServlet):
             logger.exception("Failed to remove threepid")
             raise SynapseError(500, "Failed to remove threepid")
 
+        if self.hs.config.shadow_server:
+            shadow_user = UserID(
+                requester.user.localpart, self.hs.config.shadow_server.get("hs")
+            )
+            self.shadow_3pid_delete(body, shadow_user.to_string())
+
         if ret:
             id_server_unbind_result = "success"
         else:
@@ -399,6 +468,78 @@ class ThreepidDeleteRestServlet(RestServlet):
             "id_server_unbind_result": id_server_unbind_result,
         }))
 
+    @defer.inlineCallbacks
+    def shadow_3pid_delete(self, body, user_id):
+        # TODO: retries
+        shadow_hs_url = self.hs.config.shadow_server.get("hs_url")
+        as_token = self.hs.config.shadow_server.get("as_token")
+
+        yield self.http_client.post_json_get_json(
+            "%s/_matrix/client/r0/account/3pid/delete?access_token=%s&user_id=%s" % (
+                shadow_hs_url, as_token, user_id
+            ),
+            body
+        )
+
+
+class ThreepidLookupRestServlet(RestServlet):
+    PATTERNS = [re.compile("^/_matrix/client/unstable/account/3pid/lookup$")]
+
+    def __init__(self, hs):
+        super(ThreepidLookupRestServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.identity_handler = hs.get_handlers().identity_handler
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        """Proxy a /_matrix/identity/api/v1/lookup request to an identity
+        server
+        """
+        yield self.auth.get_user_by_req(request)
+
+        # Verify query parameters
+        query_params = request.args
+        assert_params_in_dict(query_params, [b"medium", b"address", b"id_server"])
+
+        # Retrieve needed information from query parameters
+        medium = parse_string(request, "medium")
+        address = parse_string(request, "address")
+        id_server = parse_string(request, "id_server")
+
+        # Proxy the request to the identity server. lookup_3pid handles checking
+        # if the lookup is allowed so we don't need to do it here.
+        ret = yield self.identity_handler.lookup_3pid(id_server, medium, address)
+
+        defer.returnValue((200, ret))
+
+
+class ThreepidBulkLookupRestServlet(RestServlet):
+    PATTERNS = [re.compile("^/_matrix/client/unstable/account/3pid/bulk_lookup$")]
+
+    def __init__(self, hs):
+        super(ThreepidBulkLookupRestServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.identity_handler = hs.get_handlers().identity_handler
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        """Proxy a /_matrix/identity/api/v1/bulk_lookup request to an identity
+        server
+        """
+        yield self.auth.get_user_by_req(request)
+
+        body = parse_json_object_from_request(request)
+
+        assert_params_in_dict(body, ["threepids", "id_server"])
+
+        # Proxy the request to the identity server. lookup_3pid handles checking
+        # if the lookup is allowed so we don't need to do it here.
+        ret = yield self.identity_handler.bulk_lookup_3pid(
+            body["id_server"], body["threepids"],
+        )
+
+        defer.returnValue((200, ret))
+
 
 class WhoamiRestServlet(RestServlet):
     PATTERNS = client_v2_patterns("/account/whoami$")
@@ -423,4 +564,6 @@ def register_servlets(hs, http_server):
     MsisdnThreepidRequestTokenRestServlet(hs).register(http_server)
     ThreepidRestServlet(hs).register(http_server)
     ThreepidDeleteRestServlet(hs).register(http_server)
+    ThreepidLookupRestServlet(hs).register(http_server)
+    ThreepidBulkLookupRestServlet(hs).register(http_server)
     WhoamiRestServlet(hs).register(http_server)

synapse/rest/client/v2_alpha/account_data.py

@@ -19,6 +19,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import AuthError, NotFoundError, SynapseError
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.types import UserID
 
 from ._base import client_v2_patterns
 
@@ -39,6 +40,7 @@ class AccountDataServlet(RestServlet):
         self.auth = hs.get_auth()
         self.store = hs.get_datastore()
         self.notifier = hs.get_notifier()
+        self._profile_handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id, account_data_type):
@@ -48,6 +50,11 @@ class AccountDataServlet(RestServlet):
 
         body = parse_json_object_from_request(request)
 
+        if account_data_type == "im.vector.hide_profile":
+            user = UserID.from_string(user_id)
+            hide_profile = body.get('hide_profile')
+            yield self._profile_handler.set_active(user, not hide_profile, True)
+
         max_id = yield self.store.add_account_data_for_user(
             user_id, account_data_type, body
         )

synapse/rest/client/v2_alpha/password_policy.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+logger = logging.getLogger(__name__)
+
+
+class PasswordPolicyServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/password_policy$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(PasswordPolicyServlet, self).__init__()
+
+        self.policy = hs.config.password_policy
+        self.enabled = hs.config.password_policy_enabled
+
+    def on_GET(self, request):
+        if not self.enabled or not self.policy:
+            return (200, {})
+
+        policy = {}
+
+        for param in [
+            "minimum_length",
+            "require_digit",
+            "require_symbol",
+            "require_lowercase",
+            "require_uppercase",
+        ]:
+            if param in self.policy:
+                policy["m.%s" % param] = self.policy[param]
+
+        return (200, policy)
+
+
+def register_servlets(hs, http_server):
+    PasswordPolicyServlet(hs).register(http_server)

synapse/rest/client/v2_alpha/register.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015 - 2016 OpenMarket Ltd
-# Copyright 2017 Vector Creations Ltd
+# Copyright 2015-2016 OpenMarket Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +17,9 @@
 
 import hmac
 import logging
+import re
 from hashlib import sha1
+from string import capwords
 
 from six import string_types
 
@@ -79,10 +82,10 @@ class EmailRegisterRequestTokenRestServlet(RestServlet):
             'id_server', 'client_secret', 'email', 'send_attempt'
         ])
 
-        if not check_3pid_allowed(self.hs, "email", body['email']):
+        if not (yield check_3pid_allowed(self.hs, "email", body['email'])):
             raise SynapseError(
                 403,
-                "Your email domain is not authorized to register on this server",
+                "Your email is not authorized to register on this server",
                 Codes.THREEPID_DENIED,
             )
 
@@ -121,7 +124,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
-        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+        if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)):
             raise SynapseError(
                 403,
                 "Phone numbers are not authorized to register on this server",
@@ -200,6 +203,7 @@ class RegisterRestServlet(RestServlet):
         self.room_member_handler = hs.get_room_member_handler()
         self.macaroon_gen = hs.get_macaroon_generator()
         self.ratelimiter = hs.get_registration_ratelimiter()
+        self.password_policy_handler = hs.get_password_policy_handler()
         self.clock = hs.get_clock()
 
     @interactive_auth_handler
@@ -243,6 +247,7 @@ class RegisterRestServlet(RestServlet):
             if (not isinstance(body['password'], string_types) or
                     len(body['password']) > 512):
                 raise SynapseError(400, "Invalid password")
+            self.password_policy_handler.validate_password(body['password'])
             desired_password = body["password"]
 
         desired_username = None
@@ -252,6 +257,8 @@ class RegisterRestServlet(RestServlet):
                 raise SynapseError(400, "Invalid username")
             desired_username = body['username']
 
+        desired_display_name = body.get('display_name')
+
         appservice = None
         if self.auth.has_access_token(request):
             appservice = yield self.auth.get_appservice_by_req(request)
@@ -275,7 +282,8 @@ class RegisterRestServlet(RestServlet):
 
             if isinstance(desired_username, string_types):
                 result = yield self._do_appservice_registration(
-                    desired_username, access_token, body
+                    desired_username, desired_password, desired_display_name,
+                    access_token, body
                 )
             defer.returnValue((200, result))  # we throw for non 200 responses
             return
@@ -401,7 +409,7 @@ class RegisterRestServlet(RestServlet):
                     medium = auth_result[login_type]['medium']
                     address = auth_result[login_type]['address']
 
-                    if not check_3pid_allowed(self.hs, medium, address):
+                    if not (yield check_3pid_allowed(self.hs, medium, address)):
                         raise SynapseError(
                             403,
                             "Third party identifiers (email/phone numbers)" +
@@ -409,6 +417,95 @@ class RegisterRestServlet(RestServlet):
                             Codes.THREEPID_DENIED,
                         )
 
+                    existingUid = yield self.store.get_user_id_by_threepid(
+                        medium, address,
+                    )
+
+                    if existingUid is not None:
+                        raise SynapseError(
+                            400,
+                            "%s is already in use" % medium,
+                            Codes.THREEPID_IN_USE,
+                        )
+
+        if self.hs.config.register_mxid_from_3pid:
+            # override the desired_username based on the 3PID if any.
+            # reset it first to avoid folks picking their own username.
+            desired_username = None
+
+            # we should have an auth_result at this point if we're going to progress
+            # to register the user (i.e. we haven't picked up a registered_user_id
+            # from our session store), in which case get ready and gen the
+            # desired_username
+            if auth_result:
+                if (
+                    self.hs.config.register_mxid_from_3pid == 'email' and
+                    LoginType.EMAIL_IDENTITY in auth_result
+                ):
+                    address = auth_result[LoginType.EMAIL_IDENTITY]['address']
+                    desired_username = synapse.types.strip_invalid_mxid_characters(
+                        address.replace('@', '-').lower()
+                    )
+
+                    # find a unique mxid for the account, suffixing numbers
+                    # if needed
+                    while True:
+                        try:
+                            yield self.registration_handler.check_username(
+                                desired_username,
+                                guest_access_token=guest_access_token,
+                                assigned_user_id=registered_user_id,
+                            )
+                            # if we got this far we passed the check.
+                            break
+                        except SynapseError as e:
+                            if e.errcode == Codes.USER_IN_USE:
+                                m = re.match(r'^(.*?)(\d+)$', desired_username)
+                                if m:
+                                    desired_username = m.group(1) + str(
+                                        int(m.group(2)) + 1
+                                    )
+                                else:
+                                    desired_username += "1"
+                            else:
+                                # something else went wrong.
+                                break
+
+                    if self.hs.config.register_just_use_email_for_display_name:
+                        desired_display_name = address
+                    else:
+                        # XXX: a nasty heuristic to turn an email address into
+                        # a displayname, as part of register_mxid_from_3pid
+                        parts = address.replace('.', ' ').split('@')
+                        org_parts = parts[1].split(' ')
+
+                        if org_parts[-2] == "matrix" and org_parts[-1] == "org":
+                            org = "Tchap Admin"
+                        elif org_parts[-2] == "gouv" and org_parts[-1] == "fr":
+                            org = org_parts[-3] if len(org_parts) > 2 else org_parts[-2]
+                        else:
+                            org = org_parts[-2]
+
+                        desired_display_name = (
+                            capwords(parts[0]) + " [" + capwords(org) + "]"
+                        )
+                elif (
+                    self.hs.config.register_mxid_from_3pid == 'msisdn' and
+                    LoginType.MSISDN in auth_result
+                ):
+                    desired_username = auth_result[LoginType.MSISDN]['address']
+                else:
+                    raise SynapseError(
+                        400, "Cannot derive mxid from 3pid; no recognised 3pid"
+                    )
+
+        if desired_username is not None:
+            yield self.registration_handler.check_username(
+                desired_username,
+                guest_access_token=guest_access_token,
+                assigned_user_id=registered_user_id,
+            )
+
         if registered_user_id is not None:
             logger.info(
                 "Already registered user ID %r for this session",
@@ -420,9 +517,16 @@ class RegisterRestServlet(RestServlet):
             # NB: This may be from the auth handler and NOT from the POST
             assert_params_in_dict(params, ["password"])
 
-            desired_username = params.get("username", None)
+            if not self.hs.config.register_mxid_from_3pid:
+                desired_username = params.get("username", None)
+            else:
+                # we keep the original desired_username derived from the 3pid above
+                pass
+
             guest_access_token = params.get("guest_access_token", None)
-            new_password = params.get("password", None)
+
+            # XXX: don't we need to validate these for length etc like we did on
+            # the ones from the JSON body earlier on in the method?
 
             if desired_username is not None:
                 desired_username = desired_username.lower()
@@ -455,9 +559,10 @@ class RegisterRestServlet(RestServlet):
 
             (registered_user_id, _) = yield self.registration_handler.register(
                 localpart=desired_username,
-                password=new_password,
+                password=params.get("password", None),
                 guest_access_token=guest_access_token,
                 generate_token=False,
+                default_display_name=desired_display_name,
                 threepid=threepid,
                 address=client_addr,
             )
@@ -469,6 +574,14 @@ class RegisterRestServlet(RestServlet):
                 ):
                     yield self.store.upsert_monthly_active_user(registered_user_id)
 
+            if self.hs.config.shadow_server:
+                yield self.registration_handler.shadow_register(
+                    localpart=desired_username,
+                    display_name=desired_display_name,
+                    auth_result=auth_result,
+                    params=params,
+                )
+
             # remember that we've now registered that user account, and with
             #  what user ID (since the user may not have specified)
             self.auth_handler.set_session_data(
@@ -496,11 +609,33 @@ class RegisterRestServlet(RestServlet):
         return 200, {}
 
     @defer.inlineCallbacks
-    def _do_appservice_registration(self, username, as_token, body):
+    def _do_appservice_registration(
+        self, username, password, display_name, as_token, body
+    ):
+
+        # FIXME: appservice_register() is horribly duplicated with register()
+        # and they should probably just be combined together with a config flag.
         user_id = yield self.registration_handler.appservice_register(
-            username, as_token
+            username, as_token, password, display_name
         )
-        defer.returnValue((yield self._create_registration_details(user_id, body)))
+        result = yield self._create_registration_details(user_id, body)
+
+        auth_result = body.get('auth_result')
+        if auth_result and LoginType.EMAIL_IDENTITY in auth_result:
+            threepid = auth_result[LoginType.EMAIL_IDENTITY]
+            yield self._register_email_threepid(
+                user_id, threepid, result["access_token"],
+                body.get("bind_email")
+            )
+
+        if auth_result and LoginType.MSISDN in auth_result:
+            threepid = auth_result[LoginType.MSISDN]
+            yield self._register_msisdn_threepid(
+                user_id, threepid, result["access_token"],
+                body.get("bind_msisdn")
+            )
+
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def _do_shared_secret_registration(self, username, password, body):

synapse/rest/client/v2_alpha/user_directory.py

@@ -15,6 +15,8 @@
 
 import logging
 
+from signedjson.sign import sign_json
+
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
@@ -37,6 +39,7 @@ class UserDirectorySearchRestServlet(RestServlet):
         self.hs = hs
         self.auth = hs.get_auth()
         self.user_directory_handler = hs.get_user_directory_handler()
+        self.http_client = hs.get_simple_http_client()
 
     @defer.inlineCallbacks
     def on_POST(self, request):
@@ -67,6 +70,14 @@ class UserDirectorySearchRestServlet(RestServlet):
 
         body = parse_json_object_from_request(request)
 
+        if self.hs.config.user_directory_defer_to_id_server:
+            signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0])
+            url = "%s/_matrix/identity/api/v1/user_directory/search" % (
+                self.hs.config.user_directory_defer_to_id_server,
+            )
+            resp = yield self.http_client.post_json_get_json(url, signed_body)
+            defer.returnValue((200, resp))
+
         limit = body.get("limit", 10)
         limit = min(limit, 50)
 

synapse/rulecheck/domain_rule_checker.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from synapse.config._base import ConfigError
+
+logger = logging.getLogger(__name__)
+
+
+class DomainRuleChecker(object):
+    """
+    A re-implementation of the SpamChecker that prevents users in one domain from
+    inviting users in other domains to rooms, based on a configuration.
+
+    Takes a config in the format:
+
+    spam_checker:
+        module: "rulecheck.DomainRuleChecker"
+        config:
+          domain_mapping:
+            "inviter_domain": [ "invitee_domain_permitted", "other_domain_permitted" ]
+            "other_inviter_domain": [ "invitee_domain_permitted" ]
+          default: False
+
+          # Only let local users join rooms if they were explicitly invited.
+          can_only_join_rooms_with_invite: false
+
+          # Only let local users create rooms if they are inviting only one
+          # other user, and that user matches the rules above.
+          can_only_create_one_to_one_rooms: false
+
+          # Only let local users invite during room creation, regardless of the
+          # domain mapping rules above.
+          can_only_invite_during_room_creation: false
+
+          # Prevent local users from inviting users from certain domains to
+          # rooms published in the room directory.
+          domains_prevented_from_being_invited_to_published_rooms: []
+
+          # Allow third party invites
+          can_invite_by_third_party_id: true
+
+    Don't forget to consider if you can invite users from your own domain.
+    """
+
+    def __init__(self, config):
+        self.domain_mapping = config["domain_mapping"] or {}
+        self.default = config["default"]
+
+        self.can_only_join_rooms_with_invite = config.get(
+            "can_only_join_rooms_with_invite", False,
+        )
+        self.can_only_create_one_to_one_rooms = config.get(
+            "can_only_create_one_to_one_rooms", False,
+        )
+        self.can_only_invite_during_room_creation = config.get(
+            "can_only_invite_during_room_creation", False,
+        )
+        self.can_invite_by_third_party_id = config.get(
+            "can_invite_by_third_party_id", True,
+        )
+        self.domains_prevented_from_being_invited_to_published_rooms = config.get(
+            "domains_prevented_from_being_invited_to_published_rooms", [],
+        )
+
+    def check_event_for_spam(self, event):
+        """Implements synapse.events.SpamChecker.check_event_for_spam
+        """
+        return False
+
+    def user_may_invite(self, inviter_userid, invitee_userid, third_party_invite,
+                        room_id, new_room, published_room=False):
+        """Implements synapse.events.SpamChecker.user_may_invite
+        """
+        if self.can_only_invite_during_room_creation and not new_room:
+            return False
+
+        if not self.can_invite_by_third_party_id and third_party_invite:
+            return False
+
+        # This is a third party invite (without a bound mxid), so unless we have
+        # banned all third party invites (above) we allow it.
+        if not invitee_userid:
+            return True
+
+        inviter_domain = self._get_domain_from_id(inviter_userid)
+        invitee_domain = self._get_domain_from_id(invitee_userid)
+
+        if inviter_domain not in self.domain_mapping:
+            return self.default
+
+        if (
+            published_room and
+            invitee_domain in self.domains_prevented_from_being_invited_to_published_rooms
+        ):
+            return False
+
+        return invitee_domain in self.domain_mapping[inviter_domain]
+
+    def user_may_create_room(self, userid, invite_list, third_party_invite_list,
+                             cloning):
+        """Implements synapse.events.SpamChecker.user_may_create_room
+        """
+
+        if cloning:
+            return True
+
+        if not self.can_invite_by_third_party_id and third_party_invite_list:
+            return False
+
+        number_of_invites = len(invite_list) + len(third_party_invite_list)
+
+        if self.can_only_create_one_to_one_rooms and number_of_invites != 1:
+            return False
+
+        return True
+
+    def user_may_create_room_alias(self, userid, room_alias):
+        """Implements synapse.events.SpamChecker.user_may_create_room_alias
+        """
+        return True
+
+    def user_may_publish_room(self, userid, room_id):
+        """Implements synapse.events.SpamChecker.user_may_publish_room
+        """
+        return True
+
+    def user_may_join_room(self, userid, room_id, is_invited):
+        """Implements synapse.events.SpamChecker.user_may_join_room
+        """
+        if self.can_only_join_rooms_with_invite and not is_invited:
+            return False
+
+        return True
+
+    @staticmethod
+    def parse_config(config):
+        """Implements synapse.events.SpamChecker.parse_config
+        """
+        if "default" in config:
+            return config
+        else:
+            raise ConfigError("No default set for spam_config DomainRuleChecker")
+
+    @staticmethod
+    def _get_domain_from_id(mxid):
+        """Parses a string and returns the domain part of the mxid.
+
+        Args:
+           mxid (str): a valid mxid
+
+        Returns:
+           str: the domain part of the mxid
+
+        """
+        idx = mxid.find(":")
+        if idx == -1:
+            raise Exception("Invalid ID: %r" % (mxid,))
+        return mxid[idx + 1:]

synapse/server.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -62,6 +64,7 @@ from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.handlers.initial_sync import InitialSyncHandler
 from synapse.handlers.message import EventCreationHandler, MessageHandler
 from synapse.handlers.pagination import PaginationHandler
+from synapse.handlers.password_policy import PasswordPolicyHandler
 from synapse.handlers.presence import PresenceHandler
 from synapse.handlers.profile import BaseProfileHandler, MasterProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
@@ -187,6 +190,7 @@ class HomeServer(object):
         'registration_handler',
         'account_validity_handler',
         'event_client_serializer',
+        'password_policy_handler',
     ]
 
     REQUIRED_ON_MASTER_STARTUP = [
@@ -516,6 +520,9 @@ class HomeServer(object):
     def build_event_client_serializer(self):
         return EventClientSerializer(self)
 
+    def build_password_policy_handler(self):
+        return PasswordPolicyHandler(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 

synapse/storage/_base.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 import itertools
 import logging
+import random
 import sys
 import threading
 import time
@@ -247,6 +248,8 @@ class SQLBaseStore(object):
                 self._check_safe_to_upsert,
             )
 
+        self.rand = random.SystemRandom()
+
         if self._account_validity.enabled:
             self._clock.call_later(
                 0.0,
@@ -308,21 +311,36 @@ class SQLBaseStore(object):
             res = self.cursor_to_dict(txn)
             if res:
                 for user in res:
-                    self.set_expiration_date_for_user_txn(txn, user["name"])
+                    self.set_expiration_date_for_user_txn(
+                        txn,
+                        user["name"],
+                        use_delta=True,
+                    )
 
         yield self.runInteraction(
             "get_users_with_no_expiration_date",
             select_users_with_no_expiration_date_txn,
         )
 
-    def set_expiration_date_for_user_txn(self, txn, user_id):
+    def set_expiration_date_for_user_txn(self, txn, user_id, use_delta=False):
         """Sets an expiration date to the account with the given user ID.
 
         Args:
              user_id (str): User ID to set an expiration date for.
+             use_delta (bool): If set to False, the expiration date for the user will be
+                now + validity period. If set to True, this expiration date will be a
+                random value in the [now + period - d ; now + period] range, d being a
+                delta equal to 10% of the validity period.
         """
         now_ms = self._clock.time_msec()
         expiration_ts = now_ms + self._account_validity.period
+
+        if use_delta:
+            expiration_ts = self.rand.randrange(
+                expiration_ts - self._account_validity.startup_job_max_delta,
+                expiration_ts,
+            )
+
         self._simple_insert_txn(
             txn,
             "account_validity",

synapse/storage/appservice.py

@@ -35,7 +35,7 @@ def _make_exclusive_regex(services_cache):
     exclusive_user_regexes = [
         regex.pattern
         for service in services_cache
-        for regex in service.get_exlusive_user_regexes()
+        for regex in service.get_exclusive_user_regexes()
     ]
     if exclusive_user_regexes:
         exclusive_user_regex = "|".join("(" + r + ")" for r in exclusive_user_regexes)

synapse/storage/profile.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,6 +21,8 @@ from synapse.storage.roommember import ProfileInfo
 
 from ._base import SQLBaseStore
 
+BATCH_SIZE = 100
+
 
 class ProfileWorkerStore(SQLBaseStore):
     @defer.inlineCallbacks
@@ -61,6 +64,55 @@ class ProfileWorkerStore(SQLBaseStore):
             desc="get_profile_avatar_url",
         )
 
+    def get_latest_profile_replication_batch_number(self):
+        def f(txn):
+            txn.execute("SELECT MAX(batch) as maxbatch FROM profiles")
+            rows = self.cursor_to_dict(txn)
+            return rows[0]['maxbatch']
+        return self.runInteraction(
+            "get_latest_profile_replication_batch_number", f,
+        )
+
+    def get_profile_batch(self, batchnum):
+        return self._simple_select_list(
+            table="profiles",
+            keyvalues={
+                "batch": batchnum,
+            },
+            retcols=("user_id", "displayname", "avatar_url", "active"),
+            desc="get_profile_batch",
+        )
+
+    def assign_profile_batch(self):
+        def f(txn):
+            sql = (
+                "UPDATE profiles SET batch = "
+                "(SELECT COALESCE(MAX(batch), -1) + 1 FROM profiles) "
+                "WHERE user_id in ("
+                "    SELECT user_id FROM profiles WHERE batch is NULL limit ?"
+                ")"
+            )
+            txn.execute(sql, (BATCH_SIZE,))
+            return txn.rowcount
+        return self.runInteraction("assign_profile_batch", f)
+
+    def get_replication_hosts(self):
+        def f(txn):
+            txn.execute("SELECT host, last_synced_batch FROM profile_replication_status")
+            rows = self.cursor_to_dict(txn)
+            return {r['host']: r['last_synced_batch'] for r in rows}
+        return self.runInteraction("get_replication_hosts", f)
+
+    def update_replication_batch_for_host(self, host, last_synced_batch):
+        return self._simple_upsert(
+            table="profile_replication_status",
+            keyvalues={"host": host},
+            values={
+                "last_synced_batch": last_synced_batch,
+            },
+            desc="update_replication_batch_for_host",
+        )
+
     def get_from_remote_profile_cache(self, user_id):
         return self._simple_select_one(
             table="remote_profile_cache",
@@ -70,25 +122,46 @@ class ProfileWorkerStore(SQLBaseStore):
             desc="get_from_remote_profile_cache",
         )
 
-    def create_profile(self, user_localpart):
-        return self._simple_insert(
-            table="profiles", values={"user_id": user_localpart}, desc="create_profile"
-        )
-
-    def set_profile_displayname(self, user_localpart, new_displayname):
-        return self._simple_update_one(
+    def set_profile_displayname(self, user_localpart, new_displayname, batchnum):
+        return self._simple_upsert(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname},
+            values={
+                "displayname": new_displayname,
+                "batch": batchnum,
+            },
             desc="set_profile_displayname",
+            lock=False  # we can do this because user_id has a unique index
         )
 
-    def set_profile_avatar_url(self, user_localpart, new_avatar_url):
-        return self._simple_update_one(
+    def set_profile_avatar_url(self, user_localpart, new_avatar_url, batchnum):
+        return self._simple_upsert(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"avatar_url": new_avatar_url},
+            values={
+                "avatar_url": new_avatar_url,
+                "batch": batchnum,
+            },
             desc="set_profile_avatar_url",
+            lock=False  # we can do this because user_id has a unique index
+        )
+
+    def set_profile_active(self, user_localpart, active, hide, batchnum):
+        values = {
+            "active": int(active),
+            "batch": batchnum,
+        }
+        if not active and not hide:
+            # we are deactivating for real (not in hide mode)
+            # so clear the profile.
+            values["avatar_url"] = None
+            values["displayname"] = None
+        return self._simple_upsert(
+            table="profiles",
+            keyvalues={"user_id": user_localpart},
+            values=values,
+            desc="set_profile_active",
+            lock=False  # we can do this because user_id has a unique index
         )
 
 

synapse/storage/room.py

@@ -171,6 +171,24 @@ class RoomWorkerStore(SQLBaseStore):
             desc="is_room_blocked",
         )
 
+    @defer.inlineCallbacks
+    def is_room_published(self, room_id):
+        """Check whether a room has been published in the local public room
+        directory.
+
+        Args:
+            room_id (str)
+        Returns:
+            bool: Whether the room is currently published in the room directory
+        """
+        # Get room information
+        room_info = yield self.get_room(room_id)
+        if not room_info:
+            defer.returnValue(False)
+
+        # Check the is_public value
+        defer.returnValue(room_info.get("is_public", False))
+
     @cachedInlineCallbacks(max_entries=10000)
     def get_ratelimit_for_user(self, user_id):
         """Check if there are any overrides for ratelimiting for the given

synapse/storage/schema/delta/48/profiles_batch.sql

@@ -0,0 +1,36 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Add a batch number to track changes to profiles and the
+ * order they're made in so we can replicate user profiles
+ * to other hosts as they change
+ */
+ALTER TABLE profiles ADD COLUMN batch BIGINT DEFAULT NULL;
+
+/*
+ * Index on the batch number so we can get profiles
+ * by their batch
+ */
+CREATE INDEX profiles_batch_idx ON profiles(batch);
+
+/*
+ * A table to track what batch of user profiles has been
+ * synced to what profile replication target.
+ */
+CREATE TABLE profile_replication_status (
+    host TEXT NOT NULL,
+    last_synced_batch BIGINT NOT NULL
+);

synapse/storage/schema/delta/50/profiles_deactivated_users.sql

@@ -0,0 +1,23 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * A flag saying whether the user owning the profile has been deactivated
+ * This really belongs on the users table, not here, but the users table
+ * stores users by their full user_id and profiles stores them by localpart,
+ * so we can't easily join between the two tables. Plus, the batch number
+ * realy ought to represent data in this table that has changed.
+ */
+ALTER TABLE profiles ADD COLUMN active SMALLINT DEFAULT 1 NOT NULL;

synapse/types.py

@@ -231,6 +231,18 @@ def contains_invalid_mxid_characters(localpart):
     return any(c not in mxid_localpart_allowed_characters for c in localpart)
 
 
+def strip_invalid_mxid_characters(localpart):
+    """Removes any invalid characters from an mxid
+
+    Args:
+        localpart (basestring): the localpart to be stripped
+
+    Returns:
+        localpart (basestring): the localpart having been stripped
+    """
+    return filter(lambda c: c in mxid_localpart_allowed_characters, localpart)
+
+
 UPPER_CASE_PATTERN = re.compile(b"[A-Z_]")
 
 # the following is a pattern which matches '=', and bytes which are not allowed in a mxid

synapse/util/threepids.py

@@ -16,11 +16,14 @@
 import logging
 import re
 
+from twisted.internet import defer
+
 logger = logging.getLogger(__name__)
 
 
+@defer.inlineCallbacks
 def check_3pid_allowed(hs, medium, address):
-    """Checks whether a given format of 3PID is allowed to be used on this HS
+    """Checks whether a given 3PID is allowed to be used on this HS
 
     Args:
         hs (synapse.server.HomeServer): server
@@ -28,9 +31,35 @@ def check_3pid_allowed(hs, medium, address):
         address (str): address within that medium (e.g. "wotan@matrix.org")
             msisdns need to first have been canonicalised
     Returns:
-        bool: whether the 3PID medium/address is allowed to be added to this HS
+        defered bool: whether the 3PID medium/address is allowed to be added to this HS
     """
 
+    if hs.config.check_is_for_allowed_local_3pids:
+        data = yield hs.get_simple_http_client().get_json(
+            "https://%s%s" % (
+                hs.config.check_is_for_allowed_local_3pids,
+                "/_matrix/identity/api/v1/internal-info"
+            ),
+            {'medium': medium, 'address': address}
+        )
+
+        # Check for invalid response
+        if 'hs' not in data and 'shadow_hs' not in data:
+            defer.returnValue(False)
+
+        # Check if this user is intended to register for this homeserver
+        if (
+            data.get('hs') != hs.config.server_name
+            and data.get('shadow_hs') != hs.config.server_name
+        ):
+            defer.returnValue(False)
+
+        if data.get('requires_invite', False) and not data.get('invited', False):
+            # Requires an invite but hasn't been invited
+            defer.returnValue(False)
+
+        defer.returnValue(True)
+
     if hs.config.allowed_local_3pids:
         for constraint in hs.config.allowed_local_3pids:
             logger.debug(
@@ -41,8 +70,8 @@ def check_3pid_allowed(hs, medium, address):
                 medium == constraint['medium'] and
                 re.match(constraint['pattern'], address)
             ):
-                return True
+                defer.returnValue(True)
     else:
-        return True
+        defer.returnValue(True)
 
-    return False
+    defer.returnValue(False)

tests/handlers/test_profile.py

@@ -67,13 +67,13 @@ class ProfileTestCase(unittest.TestCase):
         self.bob = UserID.from_string("@4567:test")
         self.alice = UserID.from_string("@alice:remote")
 
-        yield self.store.create_profile(self.frank.localpart)
-
         self.handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def test_get_my_name(self):
-        yield self.store.set_profile_displayname(self.frank.localpart, "Frank")
+        yield self.store.set_profile_displayname(
+            self.frank.localpart, "Frank", 1,
+        )
 
         displayname = yield self.handler.get_displayname(self.frank)
 
@@ -116,8 +116,7 @@ class ProfileTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_incoming_fed_query(self):
-        yield self.store.create_profile("caroline")
-        yield self.store.set_profile_displayname("caroline", "Caroline")
+        yield self.store.set_profile_displayname("caroline", "Caroline", 1)
 
         response = yield self.query_handlers["profile"](
             {"user_id": "@caroline:test", "field": "displayname"}
@@ -128,7 +127,7 @@ class ProfileTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_get_my_avatar(self):
         yield self.store.set_profile_avatar_url(
-            self.frank.localpart, "http://my.server/me.png"
+            self.frank.localpart, "http://my.server/me.png", 1,
         )
 
         avatar_url = yield self.handler.get_avatar_url(self.frank)

tests/rest/client/test_identity.py

@@ -15,15 +15,22 @@
 
 import json
 
+from mock import Mock
+
+from twisted.internet import defer
+
 import synapse.rest.admin
 from synapse.rest.client.v1 import login, room
+from synapse.rest.client.v2_alpha import account
 
 from tests import unittest
 
 
-class IdentityTestCase(unittest.HomeserverTestCase):
+class IdentityDisabledTestCase(unittest.HomeserverTestCase):
+    """Tests that 3PID lookup attempts fail when the HS's config disallows them."""
 
     servlets = [
+        account.register_servlets,
         synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
@@ -32,19 +39,110 @@ class IdentityTestCase(unittest.HomeserverTestCase):
     def make_homeserver(self, reactor, clock):
 
         config = self.default_config()
+        config["trusted_third_party_id_servers"] = [
+            "testis",
+        ]
         config["enable_3pid_lookup"] = False
         self.hs = self.setup_test_homeserver(config=config)
 
         return self.hs
 
-    def test_3pid_lookup_disabled(self):
-        self.hs.config.enable_3pid_lookup = False
-
-        self.register_user("kermit", "monkey")
-        tok = self.login("kermit", "monkey")
+    def prepare(self, reactor, clock, hs):
+        self.user_id = self.register_user("kermit", "monkey")
+        self.tok = self.login("kermit", "monkey")
 
+    def test_3pid_invite_disabled(self):
         request, channel = self.make_request(
-            b"POST", "/createRoom", b"{}", access_token=tok
+            b"POST", "/createRoom", b"{}", access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+        room_id = channel.json_body["room_id"]
+
+        params = {
+            "id_server": "testis",
+            "medium": "email",
+            "address": "test@example.com",
+        }
+        request_data = json.dumps(params)
+        request_url = (
+            "/rooms/%s/invite" % (room_id)
+        ).encode('ascii')
+        request, channel = self.make_request(
+            b"POST", request_url, request_data, access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"403", channel.result)
+
+    def test_3pid_lookup_disabled(self):
+        url = ("/_matrix/client/unstable/account/3pid/lookup"
+               "?id_server=testis&medium=email&address=foo@bar.baz")
+        request, channel = self.make_request("GET", url, access_token=self.tok)
+        self.render(request)
+        self.assertEqual(channel.result["code"], b"403", channel.result)
+
+    def test_3pid_bulk_lookup_disabled(self):
+        url = "/_matrix/client/unstable/account/3pid/bulk_lookup"
+        data = {
+            "id_server": "testis",
+            "threepids": [
+                [
+                    "email",
+                    "foo@bar.baz"
+                ],
+                [
+                    "email",
+                    "john.doe@matrix.org"
+                ]
+            ]
+        }
+        request_data = json.dumps(data)
+        request, channel = self.make_request(
+            "POST", url, request_data, access_token=self.tok,
+        )
+        self.render(request)
+        self.assertEqual(channel.result["code"], b"403", channel.result)
+
+
+class IdentityEnabledTestCase(unittest.HomeserverTestCase):
+    """Tests that 3PID lookup attempts succeed when the HS's config allows them."""
+
+    servlets = [
+        account.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+
+        config = self.default_config()
+        config["enable_3pid_lookup"] = True
+        config["trusted_third_party_id_servers"] = [
+            "testis",
+        ]
+
+        mock_http_client = Mock(spec=[
+            "get_json",
+            "post_json_get_json",
+        ])
+        mock_http_client.get_json.return_value = defer.succeed((200, "{}"))
+        mock_http_client.post_json_get_json.return_value = defer.succeed((200, "{}"))
+
+        self.hs = self.setup_test_homeserver(
+            config=config,
+            simple_http_client=mock_http_client,
+        )
+
+        return self.hs
+
+    def prepare(self, reactor, clock, hs):
+        self.user_id = self.register_user("kermit", "monkey")
+        self.tok = self.login("kermit", "monkey")
+
+    def test_3pid_invite_enabled(self):
+        request, channel = self.make_request(
+            b"POST", "/createRoom", b"{}", access_token=self.tok,
         )
         self.render(request)
         self.assertEquals(channel.result["code"], b"200", channel.result)
@@ -58,7 +156,68 @@ class IdentityTestCase(unittest.HomeserverTestCase):
         request_data = json.dumps(params)
         request_url = ("/rooms/%s/invite" % (room_id)).encode('ascii')
         request, channel = self.make_request(
-            b"POST", request_url, request_data, access_token=tok
+            b"POST", request_url, request_data, access_token=self.tok,
         )
         self.render(request)
-        self.assertEquals(channel.result["code"], b"403", channel.result)
+
+        get_json = self.hs.get_simple_http_client().get_json
+        get_json.assert_called_once_with(
+            "https://testis/_matrix/identity/api/v1/lookup",
+            {
+                "address": "test@example.com",
+                "medium": "email",
+            },
+        )
+
+    def test_3pid_lookup_enabled(self):
+        url = ("/_matrix/client/unstable/account/3pid/lookup"
+               "?id_server=testis&medium=email&address=foo@bar.baz")
+        request, channel = self.make_request("GET", url, access_token=self.tok)
+        self.render(request)
+
+        get_json = self.hs.get_simple_http_client().get_json
+        get_json.assert_called_once_with(
+            "https://testis/_matrix/identity/api/v1/lookup",
+            {
+                "address": "foo@bar.baz",
+                "medium": "email",
+            },
+        )
+
+    def test_3pid_bulk_lookup_enabled(self):
+        url = "/_matrix/client/unstable/account/3pid/bulk_lookup"
+        data = {
+            "id_server": "testis",
+            "threepids": [
+                [
+                    "email",
+                    "foo@bar.baz"
+                ],
+                [
+                    "email",
+                    "john.doe@matrix.org"
+                ]
+            ]
+        }
+        request_data = json.dumps(data)
+        request, channel = self.make_request(
+            "POST", url, request_data, access_token=self.tok,
+        )
+        self.render(request)
+
+        post_json = self.hs.get_simple_http_client().post_json_get_json
+        post_json.assert_called_once_with(
+            "https://testis/_matrix/identity/api/v1/bulk_lookup",
+            {
+                "threepids": [
+                    [
+                        "email",
+                        "foo@bar.baz"
+                    ],
+                    [
+                        "email",
+                        "john.doe@matrix.org"
+                    ]
+                ],
+            },
+        )

tests/rest/client/v2_alpha/test_password_policy.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from synapse.api.constants import LoginType
+from synapse.api.errors import Codes
+from synapse.rest import admin
+from synapse.rest.client.v1 import login
+from synapse.rest.client.v2_alpha import account, password_policy, register
+
+from tests import unittest
+
+
+class PasswordPolicyTestCase(unittest.HomeserverTestCase):
+    """Tests the password policy feature and its compliance with MSC2000.
+
+    When validating a password, Synapse does the necessary checks in this order:
+
+        1. Password is long enough
+        2. Password contains digit(s)
+        3. Password contains symbol(s)
+        4. Password contains uppercase letter(s)
+        5. Password contains lowercase letter(s)
+
+    Therefore, each test in this test case that tests whether a password triggers the
+    right error code to be returned provides a password good enough to pass the previous
+    steps but not the one it's testing (nor any step that comes after).
+    """
+
+    servlets = [
+        admin.register_servlets_for_client_rest_resource,
+        login.register_servlets,
+        register.register_servlets,
+        password_policy.register_servlets,
+        account.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        self.register_url = "/_matrix/client/r0/register"
+        self.policy = {
+            "enabled": True,
+            "minimum_length": 10,
+            "require_digit": True,
+            "require_symbol": True,
+            "require_lowercase": True,
+            "require_uppercase": True,
+        }
+
+        config = self.default_config()
+        config["password_config"] = {
+            "policy": self.policy,
+        }
+
+        hs = self.setup_test_homeserver(config=config)
+        return hs
+
+    def test_get_policy(self):
+        """Tests if the /password_policy endpoint returns the configured policy."""
+
+        request, channel = self.make_request("GET", "/_matrix/client/r0/password_policy")
+        self.render(request)
+
+        self.assertEqual(channel.code, 200, channel.result)
+        self.assertEqual(channel.json_body, {
+            "m.minimum_length": 10,
+            "m.require_digit": True,
+            "m.require_symbol": True,
+            "m.require_lowercase": True,
+            "m.require_uppercase": True,
+        }, channel.result)
+
+    def test_password_too_short(self):
+        request_data = json.dumps({"username": "kermit", "password": "shorty"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(
+            channel.json_body["errcode"],
+            Codes.PASSWORD_TOO_SHORT,
+            channel.result,
+        )
+
+    def test_password_no_digit(self):
+        request_data = json.dumps({"username": "kermit", "password": "longerpassword"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(
+            channel.json_body["errcode"],
+            Codes.PASSWORD_NO_DIGIT,
+            channel.result,
+        )
+
+    def test_password_no_symbol(self):
+        request_data = json.dumps({"username": "kermit", "password": "l0ngerpassword"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(
+            channel.json_body["errcode"],
+            Codes.PASSWORD_NO_SYMBOL,
+            channel.result,
+        )
+
+    def test_password_no_uppercase(self):
+        request_data = json.dumps({"username": "kermit", "password": "l0ngerpassword!"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(
+            channel.json_body["errcode"],
+            Codes.PASSWORD_NO_UPPERCASE,
+            channel.result,
+        )
+
+    def test_password_no_lowercase(self):
+        request_data = json.dumps({"username": "kermit", "password": "L0NGERPASSWORD!"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(
+            channel.json_body["errcode"],
+            Codes.PASSWORD_NO_LOWERCASE,
+            channel.result,
+        )
+
+    def test_password_compliant(self):
+        request_data = json.dumps({"username": "kermit", "password": "L0ngerpassword!"})
+        request, channel = self.make_request("POST", self.register_url, request_data)
+        self.render(request)
+
+        # Getting a 401 here means the password has passed validation and the server has
+        # responded with a list of registration flows.
+        self.assertEqual(channel.code, 401, channel.result)
+
+    def test_password_change(self):
+        """This doesn't test every possible use case, only that hitting /account/password
+        triggers the password validation code.
+        """
+        compliant_password = "C0mpl!antpassword"
+        not_compliant_password = "notcompliantpassword"
+
+        user_id = self.register_user("kermit", compliant_password)
+        tok = self.login("kermit", compliant_password)
+
+        request_data = json.dumps({
+            "new_password": not_compliant_password,
+            "auth": {
+                "password": compliant_password,
+                "type": LoginType.PASSWORD,
+                "user": user_id,
+            }
+        })
+        request, channel = self.make_request(
+            "POST",
+            "/_matrix/client/r0/account/password",
+            request_data,
+            access_token=tok,
+        )
+        self.render(request)
+
+        self.assertEqual(channel.code, 400, channel.result)
+        self.assertEqual(channel.json_body["errcode"], Codes.PASSWORD_NO_DIGIT)

tests/rest/client/v2_alpha/test_register.py

@@ -436,6 +436,7 @@ class AccountValidityBackgroundJobTestCase(unittest.HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
         self.validity_period = 10
+        self.max_delta = self.validity_period * 10. / 100.
 
         config = self.default_config()
 
@@ -453,14 +454,18 @@ class AccountValidityBackgroundJobTestCase(unittest.HomeserverTestCase):
 
     def test_background_job(self):
         """
-        Tests whether the account validity startup background job does the right thing,
-        which is sticking an expiration date to every account that doesn't already have
-        one.
+        Tests the same thing as test_background_job, except that it sets the
+        startup_job_max_delta parameter and checks that the expiration date is within the
+        allowed range.
         """
-        user_id = self.register_user("kermit", "user")
+        user_id = self.register_user("kermit_delta", "user")
+
+        self.hs.config.account_validity.startup_job_max_delta = self.max_delta
 
         now_ms = self.hs.clock.time_msec()
         self.get_success(self.store._set_expiration_date_when_missing())
 
         res = self.get_success(self.store.get_expiration_ts_for_user(user_id))
-        self.assertEqual(res, now_ms + self.validity_period)
+
+        self.assertGreaterEqual(res, now_ms + self.validity_period - self.max_delta)
+        self.assertLessEqual(res, now_ms + self.validity_period)

tests/rulecheck/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/rulecheck/test_domainrulecheck.py

@@ -0,0 +1,342 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+
+import synapse.rest.admin
+from synapse.config._base import ConfigError
+from synapse.rest.client.v1 import login, room
+from synapse.rulecheck.domain_rule_checker import DomainRuleChecker
+
+from tests import unittest
+from tests.server import make_request, render
+
+
+class DomainRuleCheckerTestCase(unittest.TestCase):
+    def test_allowed(self):
+        config = {
+            "default": False,
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+            },
+            "domains_prevented_from_being_invited_to_published_rooms": ["target_two"]
+        }
+        check = DomainRuleChecker(config)
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_one", "test:target_one", None, "room", False
+            )
+        )
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_one", "test:target_two", None, "room", False
+            )
+        )
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_two", "test:target_two", None, "room", False
+            )
+        )
+
+        # User can invite internal user to a published room
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_one", "test1:target_one", None, "room", False, True,
+            )
+        )
+
+        # User can invite external user to a non-published room
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_one", "test:target_two", None, "room", False, False,
+            )
+        )
+
+    def test_disallowed(self):
+        config = {
+            "default": True,
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+                "source_four": [],
+            },
+        }
+        check = DomainRuleChecker(config)
+        self.assertFalse(
+            check.user_may_invite(
+                "test:source_one", "test:target_three", None, "room", False
+            )
+        )
+        self.assertFalse(
+            check.user_may_invite(
+                "test:source_two", "test:target_three", None, "room", False
+            )
+        )
+        self.assertFalse(
+            check.user_may_invite(
+                "test:source_two", "test:target_one", None, "room", False
+            )
+        )
+        self.assertFalse(
+            check.user_may_invite(
+                "test:source_four", "test:target_one", None, "room", False
+            )
+        )
+
+        # User cannot invite external user to a published room
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_one", "test:target_two", None, "room", False, True,
+            )
+        )
+
+    def test_default_allow(self):
+        config = {
+            "default": True,
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+            },
+        }
+        check = DomainRuleChecker(config)
+        self.assertTrue(
+            check.user_may_invite(
+                "test:source_three", "test:target_one", None, "room", False
+            )
+        )
+
+    def test_default_deny(self):
+        config = {
+            "default": False,
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+            },
+        }
+        check = DomainRuleChecker(config)
+        self.assertFalse(
+            check.user_may_invite(
+                "test:source_three", "test:target_one", None, "room", False
+            )
+        )
+
+    def test_config_parse(self):
+        config = {
+            "default": False,
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+            },
+        }
+        self.assertEquals(config, DomainRuleChecker.parse_config(config))
+
+    def test_config_parse_failure(self):
+        config = {
+            "domain_mapping": {
+                "source_one": ["target_one", "target_two"],
+                "source_two": ["target_two"],
+            }
+        }
+        self.assertRaises(ConfigError, DomainRuleChecker.parse_config, config)
+
+
+class DomainRuleCheckerRoomTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    hijack_auth = False
+
+    def make_homeserver(self, reactor, clock):
+        config = self.default_config()
+        config["trusted_third_party_id_servers"] = [
+            "localhost",
+        ]
+
+        config["spam_checker"] = {
+            "module": "synapse.rulecheck.domain_rule_checker.DomainRuleChecker",
+            "config": {
+                "default": True,
+                "domain_mapping": {},
+                "can_only_join_rooms_with_invite": True,
+                "can_only_create_one_to_one_rooms": True,
+                "can_only_invite_during_room_creation": True,
+                "can_invite_by_third_party_id": False,
+            },
+        }
+
+        hs = self.setup_test_homeserver(config=config)
+        return hs
+
+    def prepare(self, reactor, clock, hs):
+        self.admin_user_id = self.register_user("admin_user", "pass", admin=True)
+        self.admin_access_token = self.login("admin_user", "pass")
+
+        self.normal_user_id = self.register_user("normal_user", "pass", admin=False)
+        self.normal_access_token = self.login("normal_user", "pass")
+
+        self.other_user_id = self.register_user("other_user", "pass", admin=False)
+
+    def test_admin_can_create_room(self):
+        channel = self._create_room(self.admin_access_token)
+        assert channel.result["code"] == b"200", channel.result
+
+    def test_normal_user_cannot_create_empty_room(self):
+        channel = self._create_room(self.normal_access_token)
+        assert channel.result["code"] == b"403", channel.result
+
+    def test_normal_user_cannot_create_room_with_multiple_invites(self):
+        channel = self._create_room(
+            self.normal_access_token,
+            content={"invite": [self.other_user_id, self.admin_user_id]},
+        )
+        assert channel.result["code"] == b"403", channel.result
+
+        # Test that it correctly counts both normal and third party invites
+        channel = self._create_room(
+            self.normal_access_token,
+            content={
+                "invite": [self.other_user_id],
+                "invite_3pid": [{"medium": "email", "address": "foo@example.com"}],
+            },
+        )
+        assert channel.result["code"] == b"403", channel.result
+
+        # Test that it correctly rejects third party invites
+        channel = self._create_room(
+            self.normal_access_token,
+            content={
+                "invite": [],
+                "invite_3pid": [{"medium": "email", "address": "foo@example.com"}],
+            },
+        )
+        assert channel.result["code"] == b"403", channel.result
+
+    def test_normal_user_can_room_with_single_invites(self):
+        channel = self._create_room(
+            self.normal_access_token, content={"invite": [self.other_user_id]}
+        )
+        assert channel.result["code"] == b"200", channel.result
+
+    def test_cannot_join_public_room(self):
+        channel = self._create_room(self.admin_access_token)
+        assert channel.result["code"] == b"200", channel.result
+
+        room_id = channel.json_body["room_id"]
+
+        self.helper.join(
+            room_id, self.normal_user_id, tok=self.normal_access_token, expect_code=403
+        )
+
+    def test_can_join_invited_room(self):
+        channel = self._create_room(self.admin_access_token)
+        assert channel.result["code"] == b"200", channel.result
+
+        room_id = channel.json_body["room_id"]
+
+        self.helper.invite(
+            room_id,
+            src=self.admin_user_id,
+            targ=self.normal_user_id,
+            tok=self.admin_access_token,
+        )
+
+        self.helper.join(
+            room_id, self.normal_user_id, tok=self.normal_access_token, expect_code=200
+        )
+
+    def test_cannot_invite(self):
+        channel = self._create_room(self.admin_access_token)
+        assert channel.result["code"] == b"200", channel.result
+
+        room_id = channel.json_body["room_id"]
+
+        self.helper.invite(
+            room_id,
+            src=self.admin_user_id,
+            targ=self.normal_user_id,
+            tok=self.admin_access_token,
+        )
+
+        self.helper.join(
+            room_id, self.normal_user_id, tok=self.normal_access_token, expect_code=200
+        )
+
+        self.helper.invite(
+            room_id,
+            src=self.normal_user_id,
+            targ=self.other_user_id,
+            tok=self.normal_access_token,
+            expect_code=403,
+        )
+
+    def test_cannot_3pid_invite(self):
+        """Test that unbound 3pid invites get rejected.
+        """
+        channel = self._create_room(self.admin_access_token)
+        assert channel.result["code"] == b"200", channel.result
+
+        room_id = channel.json_body["room_id"]
+
+        self.helper.invite(
+            room_id,
+            src=self.admin_user_id,
+            targ=self.normal_user_id,
+            tok=self.admin_access_token,
+        )
+
+        self.helper.join(
+            room_id, self.normal_user_id,
+            tok=self.normal_access_token,
+            expect_code=200,
+        )
+
+        self.helper.invite(
+            room_id,
+            src=self.normal_user_id,
+            targ=self.other_user_id,
+            tok=self.normal_access_token,
+            expect_code=403,
+        )
+
+        request, channel = self.make_request(
+            "POST",
+            "rooms/%s/invite" % (room_id),
+            {
+                "address": "foo@bar.com",
+                "medium": "email",
+                "id_server": "localhost"
+            },
+            access_token=self.normal_access_token,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 403, channel.result["body"])
+
+    def _create_room(self, token, content={}):
+        path = "/_matrix/client/r0/createRoom?access_token=%s" % (token,)
+
+        request, channel = make_request(
+            self.hs.get_reactor(),
+            "POST",
+            path,
+            content=json.dumps(content).encode("utf8"),
+        )
+        render(request, self.resource, self.hs.get_reactor())
+
+        return channel

tests/storage/test_profile.py

@@ -34,20 +34,19 @@ class ProfileStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_displayname(self):
-        yield self.store.create_profile(self.u_frank.localpart)
-
-        yield self.store.set_profile_displayname(self.u_frank.localpart, "Frank")
+        yield self.store.set_profile_displayname(
+            self.u_frank.localpart, "Frank", 1,
+        )
 
         self.assertEquals(
-            "Frank", (yield self.store.get_profile_displayname(self.u_frank.localpart))
+            "Frank",
+            (yield self.store.get_profile_displayname(self.u_frank.localpart))
         )
 
     @defer.inlineCallbacks
     def test_avatar_url(self):
-        yield self.store.create_profile(self.u_frank.localpart)
-
         yield self.store.set_profile_avatar_url(
-            self.u_frank.localpart, "http://my.site/here"
+            self.u_frank.localpart, "http://my.site/here", 1,
         )
 
         self.assertEquals(