In get_recent_events_for_room, get the full event json rather than just the event_ids so we don't have to do so many db queries

Don't perform the check in auth.is_host_in_room but instead do it in _do_join
and also assert that there are no m.room.members in the room before doing so.

CHANGES.rst

@@ -1,7 +1,21 @@
+Changes in synapse 0.6.1 (2015-01-07)
+=====================================
+
+ * Major optimizations to improve performance of initial sync and event sending
+   in large rooms (by up to 10x)
+ * Media repository now includes a Content-Length header on media downloads.
+ * Improve quality of thumbnails by changing resizing algorithm.
+
 Changes in synapse 0.6.0 (2014-12-16)
 =====================================
 
  * Add new API for media upload and download that supports thumbnailing.
+ * Replicate media uploads over multiple homeservers so media is always served
+   to clients from their local homeserver.  This obsoletes the
+   --content-addr parameter and confusion over accessing content directly
+   from remote homeservers.
+ * Implement exponential backoff when retrying federation requests when
+   sending to remote homeservers which are offline.
  * Implement typing notifications.
  * Fix bugs where we sent events with invalid signatures due to bugs where
    we incorrectly persisted events.

README.rst

@@ -108,6 +108,18 @@ To install the synapse homeserver run::
 This installs synapse, along with the libraries it uses, into
 ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX.
 
+Your python may not give priority to locally installed libraries over system
+libraries, in which case you must add your local packages to your python path::
+
+    $ # on Linux:
+    $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH
+
+    $ # on OSX:
+    $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server.  See docs/turn-howto.rst for details.
+
 Troubleshooting Installation
 ----------------------------
 
@@ -239,6 +251,11 @@ Upgrading an existing homeserver
 IMPORTANT: Before upgrading an existing homeserver to a new version, please
 refer to UPGRADE.rst for any additional instructions.
 
+Otherwise, simply re-install the new codebase over the current one - e.g.
+by ``pip install --user --process-dependency-links
+https://github.com/matrix-org/synapse/tarball/master``
+if using pip, or by ``git pull`` if running off a git working copy.
+
 
 Setting up Federation
 =====================

UPGRADE.rst

@@ -52,7 +52,7 @@ resulting conflicts during the upgrade process.
 Before running the command the homeserver should be first completely 
 shutdown. To run it, simply specify the location of the database, e.g.:
 
-  ./database-prepare-for-0.5.0.sh "homeserver.db"
+  ./scripts/database-prepare-for-0.5.0.sh "homeserver.db"
 
 Once this has successfully completed it will be safe to restart the 
 homeserver. You may notice that the homeserver takes a few seconds longer to 
@@ -147,7 +147,7 @@ rooms the home server was a member of and room alias mappings.
 Before running the command the homeserver should be first completely 
 shutdown. To run it, simply specify the location of the database, e.g.:
 
-  ./database-prepare-for-0.0.1.sh "homeserver.db"
+  ./scripts/database-prepare-for-0.0.1.sh "homeserver.db"
 
 Once this has successfully completed it will be safe to restart the 
 homeserver. You may notice that the homeserver takes a few seconds longer to 

VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1b

contrib/graph/graph2.py

@@ -23,14 +23,27 @@ import argparse
 from synapse.events import FrozenEvent
 
 
-def make_graph(db_name, room_id, file_prefix):
+def make_graph(db_name, room_id, file_prefix, limit):
     conn = sqlite3.connect(db_name)
 
-    c = conn.execute(
-        "SELECT json FROM event_json where room_id = ?",
-        (room_id,)
+    sql = (
+        "SELECT json FROM event_json as j "
+        "INNER JOIN events as e ON e.event_id = j.event_id "
+        "WHERE j.room_id = ?"
     )
 
+    args = [room_id]
+
+    if limit:
+        sql += (
+            " ORDER BY topological_ordering DESC, stream_ordering DESC "
+            "LIMIT ?"
+        )
+
+        args.append(limit)
+
+    c = conn.execute(sql, args)
+
     events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()]
 
     events.sort(key=lambda e: e.depth)
@@ -128,11 +141,16 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "-p", "--prefix", dest="prefix",
-        help="String to prefix output files with"
+        help="String to prefix output files with",
+        default="graph_output"
+    )
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
     )
     parser.add_argument('db')
     parser.add_argument('room')
 
     args = parser.parse_args()
 
-    make_graph(args.db, args.room, args.prefix)
+    make_graph(args.db, args.room, args.prefix, args.limit)

docs/media_repository.rst

@@ -1,6 +1,8 @@
-Media Repository
+Media Repository 
 ================
 
+*Synapse implementation-specific details for the media repository*
+
 The media repository is where attachments and avatar photos are stored.
 It stores attachment content and thumbnails for media uploaded by local users.
 It caches attachment content and thumbnails for media uploaded by remote users.

jsfiddles/create_room_send_msg/demo.css

@@ -1,17 +0,0 @@
-.loggedin {
-    visibility: hidden;
-}
-
-p {
-    font-family: monospace;
-}
-
-table
-{
-    border-spacing:5px;
-}
-
-th,td
-{
-    padding:5px;
-}

jsfiddles/create_room_send_msg/demo.html

@@ -1,30 +0,0 @@
-<div>
-    <p>This room creation / message sending demo requires a home server to be running on http://localhost:8008</p>
-</div>
-<form class="loginForm">
-    <input type="text" id="userLogin" placeholder="Username"></input>
-    <input type="password" id="passwordLogin" placeholder="Password"></input>
-    <input type="button" class="login" value="Login"></input>
-</form>
-<div class="loggedin">
-    <form class="createRoomForm">
-        <input type="text" id="roomAlias" placeholder="Room alias (optional)"></input>
-        <input type="button" class="createRoom" value="Create Room"></input>
-    </form>
-    <form class="sendMessageForm">
-        <input type="text" id="roomId" placeholder="Room ID"></input>
-        <input type="text" id="messageBody" placeholder="Message body"></input>
-        <input type="button" class="sendMessage" value="Send Message"></input>
-    </form>
-    <table id="rooms">
-        <tbody>
-            <tr>
-                <th>Room ID</th>
-                <th>My state</th>
-                <th>Room Alias</th>
-                <th>Latest message</th>
-            </tr>
-        </tbody>
-    </table>
-</div>
-    

jsfiddles/create_room_send_msg/demo.js

@@ -1,113 +0,0 @@
-var accountInfo = {};
-
-var showLoggedIn = function(data) {
-    accountInfo = data;
-    getCurrentRoomList();
-    $(".loggedin").css({visibility: "visible"});
-};
-
-$('.login').live('click', function() {
-    var user = $("#userLogin").val();
-    var password = $("#passwordLogin").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/login",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            showLoggedIn(data);
-        },
-        error: function(err) {
-            var errMsg = "To try this, you need a home server running!";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson) {
-                errMsg = JSON.stringify(errJson);   
-            }
-            alert(errMsg);  
-        }
-    }); 
-});
-
-var getCurrentRoomList = function() {
-    var url = "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=" + accountInfo.access_token + "&limit=1";
-    $.getJSON(url, function(data) {
-        var rooms = data.rooms;
-        for (var i=0; i<rooms.length; ++i) {
-            rooms[i].latest_message = rooms[i].messages.chunk[0].content.body;
-            addRoom(rooms[i]);   
-        }
-    }).fail(function(err) {
-        alert(JSON.stringify($.parseJSON(err.responseText)));
-    });
-};
-
-$('.createRoom').live('click', function() {
-    var roomAlias = $("#roomAlias").val();
-    var data = {};
-    if (roomAlias.length > 0) {
-        data.room_alias_name = roomAlias;   
-    }
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token="+accountInfo.access_token,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            data.membership = "join"; // you are automatically joined into every room you make.
-            data.latest_message = "";
-            addRoom(data);
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});
-
-var addRoom = function(data) {
-    row = "<tr>" +
-        "<td>"+data.room_id+"</td>" +
-        "<td>"+data.membership+"</td>" +
-        "<td>"+data.room_alias+"</td>" +
-        "<td>"+data.latest_message+"</td>" +
-        "</tr>";
-    $("#rooms").append(row);
-};
-
-$('.sendMessage').live('click', function() {
-    var roomId = $("#roomId").val();
-    var body = $("#messageBody").val();
-    var msgId = $.now();
-    
-    if (roomId.length === 0 || body.length === 0) {
-        return;
-    }
-    
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/$roomid/send/m.room.message?access_token=$token";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$roomid", encodeURIComponent(roomId));
-    
-    var data = {
-        msgtype: "m.text",
-        body: body
-    };
-    
-    $.ajax({
-        url: url,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            $("#messageBody").val("");
-            // wipe the table and reload it. Using the event stream would be the best
-            // solution but that is out of scope of this fiddle.
-            $("#rooms").find("tr:gt(0)").remove();
-            getCurrentRoomList();
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});

jsfiddles/event_stream/demo.css

@@ -1,17 +0,0 @@
-.loggedin {
-    visibility: hidden;
-}
-
-p {
-    font-family: monospace;
-}
-
-table
-{
-    border-spacing:5px;
-}
-
-th,td
-{
-    padding:5px;
-}

jsfiddles/event_stream/demo.html

@@ -1,23 +0,0 @@
-<div>
-    <p>This event stream demo requires a home server to be running on http://localhost:8008</p>
-</div>
-<form class="loginForm">
-    <input type="text" id="userLogin" placeholder="Username"></input>
-    <input type="password" id="passwordLogin" placeholder="Password"></input>
-    <input type="button" class="login" value="Login"></input>
-</form>
-<div class="loggedin">
-    <form class="sendMessageForm">
-        <input type="button" class="sendMessage" value="Send random message"></input>
-    </form>
-    <p id="streamErrorText"></p>
-    <table id="rooms">
-        <tbody>
-            <tr>
-                <th>Room ID</th>
-                <th>Latest message</th>
-            </tr>
-        </tbody>
-    </table>
-</div>
-    

jsfiddles/event_stream/demo.js

@@ -1,145 +0,0 @@
-var accountInfo = {};
-
-var eventStreamInfo = {
-    from: "END"
-};
-
-var roomInfo = [];
-
-var longpollEventStream = function() {
-    var url = "http://localhost:8008/_matrix/client/api/v1/events?access_token=$token&from=$from";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$from", eventStreamInfo.from);
-    
-    $.getJSON(url, function(data) {
-        eventStreamInfo.from = data.end;
-        
-        var hasNewLatestMessage = false;
-        for (var i=0; i<data.chunk.length; ++i) {
-            if (data.chunk[i].type === "m.room.message") {
-                for (var j=0; j<roomInfo.length; ++j) {
-                    if (roomInfo[j].room_id === data.chunk[i].room_id) {
-                        roomInfo[j].latest_message = data.chunk[i].content.body;
-                        hasNewLatestMessage = true;
-                    }
-                }
-            }
-        }
-        
-        if (hasNewLatestMessage) {
-           setRooms(roomInfo);
-        }
-        $("#streamErrorText").text("");
-        longpollEventStream();
-    }).fail(function(err) {
-        $("#streamErrorText").text("Event stream error: "+JSON.stringify($.parseJSON(err.responseText)));
-        setTimeout(longpollEventStream, 5000);
-    });
-};
-
-var showLoggedIn = function(data) {
-    accountInfo = data;
-    longpollEventStream();
-    getCurrentRoomList();
-    $(".loggedin").css({visibility: "visible"});
-};
-
-$('.login').live('click', function() {
-    var user = $("#userLogin").val();
-    var password = $("#passwordLogin").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/login",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            $("#rooms").find("tr:gt(0)").remove();
-            showLoggedIn(data);
-        },
-        error: function(err) {
-            var errMsg = "To try this, you need a home server running!";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson) {
-                errMsg = JSON.stringify(errJson);   
-            }
-            alert(errMsg);  
-        }
-    }); 
-});
-
-var getCurrentRoomList = function() {
-    $("#roomId").val("");
-    var url = "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=" + accountInfo.access_token + "&limit=1";
-    $.getJSON(url, function(data) {
-        var rooms = data.rooms;
-        for (var i=0; i<rooms.length; ++i) {
-            if ("messages" in rooms[i]) {
-                rooms[i].latest_message = rooms[i].messages.chunk[0].content.body;   
-            }
-        }
-        roomInfo = rooms;
-        setRooms(roomInfo);  
-    }).fail(function(err) {
-        alert(JSON.stringify($.parseJSON(err.responseText)));
-    });
-};
-
-$('.sendMessage').live('click', function() {
-    if (roomInfo.length === 0) {
-        alert("There is no room to send a message to!");
-        return;
-    }
-    
-    var index = Math.floor(Math.random() * roomInfo.length);
-    
-    sendMessage(roomInfo[index].room_id);
-});
-
-var sendMessage = function(roomId) {
-    var body = "jsfiddle message @" + $.now();
-    
-    if (roomId.length === 0) {
-        return;
-    }
-    
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/$roomid/send/m.room.message?access_token=$token";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$roomid", encodeURIComponent(roomId));
-    
-    var data = {
-        msgtype: "m.text",
-        body: body
-    };
-    
-    $.ajax({
-        url: url,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            $("#messageBody").val("");
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-};
-
-var setRooms = function(roomList) {
-    // wipe existing entries
-    $("#rooms").find("tr:gt(0)").remove();
-    
-    var rows = "";
-    for (var i=0; i<roomList.length; ++i) {
-        row = "<tr>" +
-            "<td>"+roomList[i].room_id+"</td>" +
-            "<td>"+roomList[i].latest_message+"</td>" +
-            "</tr>";  
-        rows += row;
-    }
-    
-    $("#rooms").append(rows);
-};
-

jsfiddles/example_app/demo.css

@@ -1,43 +0,0 @@
-.roomListDashboard, .roomContents, .sendMessageForm {
-    visibility: hidden;
-}
-
-.roomList {
-    background-color: #909090;
-}
-
-.messageWrapper {
-    background-color: #EEEEEE;
-    height: 400px;
-    overflow: scroll;
-}
-
-.membersWrapper {
-    background-color: #EEEEEE;
-    height: 200px;
-    width: 50%;
-    overflow: scroll;
-}
-
-.textEntry {
-    width: 100%
-}
-
-p {
-    font-family: monospace;
-}
-
-table
-{
-    border-spacing:5px;
-}
-
-th,td
-{
-    padding:5px;
-}
-
-.roomList tr:not(:first-child):hover {
-    background-color: orange;
-    cursor: pointer;
-}

jsfiddles/example_app/demo.details

@@ -1,7 +0,0 @@
- name: Example Matrix Client
- description: Includes login, live event streaming, creating rooms, sending messages and viewing member lists.
- authors:
-   - matrix.org
- resources:
-   - http://matrix.org
- normalize_css: no

jsfiddles/example_app/demo.html

@@ -1,56 +0,0 @@
-<div class="signUp">
-    <p>Matrix example application: Requires a local home server running at http://localhost:8008</p>
-    <form class="registrationForm">
-        <p>No account? Register:</p>
-        <input type="text" id="userReg" placeholder="Username"></input>
-        <input type="password" id="passwordReg" placeholder="Password"></input>
-        <input type="button" class="register" value="Register"></input>
-    </form>
-    <form class="loginForm">
-        <p>Got an account? Login:</p>
-        <input type="text" id="userLogin" placeholder="Username"></input>
-        <input type="password" id="passwordLogin" placeholder="Password"></input>
-        <input type="button" class="login" value="Login"></input>
-    </form>
-</div>
-
-<div class="roomListDashboard">
-    <form class="createRoomForm">
-        <input type="text" id="roomAlias" placeholder="Room alias"></input>
-        <input type="button" class="createRoom" value="Create Room"></input>
-    </form>
-    <table id="rooms" class="roomList">
-        <tbody>
-            <tr>
-                <th>Room</th>
-                <th>My state</th>
-                <th>Latest message</th>
-            </tr>
-        </tbody>
-    </table>
-</div>
-
-<div class="roomContents">
-    <p id="roomName">Select a room</p>
-    <div class="messageWrapper">
-        <table id="messages">
-            <tbody>
-            </tbody>
-        </table>
-    </div>
-    <form class="sendMessageForm">
-        <input type="text" class="textEntry" id="body" placeholder="Enter text here..." onkeydown="javascript:if (event.keyCode == 13) document.getElementById('sendMsg').focus()"></input>
-        <input type="button" class="sendMessage" id="sendMsg" value="Send"></input>
-    </form>
-</div>
-
-<div>
-    <p>Member list:</p>
-    <div class="membersWrapper">
-        <table id="members">
-            <tbody>
-            </tbody>
-        </table>
-    </div>
-</div>
-    

jsfiddles/example_app/demo.js

@@ -1,327 +0,0 @@
-var accountInfo = {};
-
-var eventStreamInfo = {
-    from: "END"
-};
-
-var roomInfo = [];
-var memberInfo = [];
-var viewingRoomId;
-
-// ************** Event Streaming **************
-var longpollEventStream = function() {
-    var url = "http://localhost:8008/_matrix/client/api/v1/events?access_token=$token&from=$from";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$from", eventStreamInfo.from);
-
-    $.getJSON(url, function(data) {
-        eventStreamInfo.from = data.end;
-        
-        var hasNewLatestMessage = false;
-        var updatedMemberList = false;
-        var i=0;
-        var j=0;
-        for (i=0; i<data.chunk.length; ++i) {
-            if (data.chunk[i].type === "m.room.message") {
-                console.log("Got new message: " + JSON.stringify(data.chunk[i]));
-                if (viewingRoomId === data.chunk[i].room_id) {
-                    addMessage(data.chunk[i]);
-                }
-                
-                for (j=0; j<roomInfo.length; ++j) {
-                    if (roomInfo[j].room_id === data.chunk[i].room_id) {
-                        roomInfo[j].latest_message = data.chunk[i].content.body;
-                        hasNewLatestMessage = true;
-                    }
-                }
-            }
-            else if (data.chunk[i].type === "m.room.member") {
-                if (viewingRoomId === data.chunk[i].room_id) {
-                    console.log("Got new member: " + JSON.stringify(data.chunk[i]));
-                    addMessage(data.chunk[i]);
-                    for (j=0; j<memberInfo.length; ++j) {
-                        if (memberInfo[j].state_key === data.chunk[i].state_key) {
-                            memberInfo[j] = data.chunk[i];
-                            updatedMemberList = true;
-                            break;
-                        }
-                    }
-                    if (!updatedMemberList) {
-                        memberInfo.push(data.chunk[i]);  
-                        updatedMemberList = true;
-                    }
-                }
-                if (data.chunk[i].state_key === accountInfo.user_id) {
-                    getCurrentRoomList(); // update our join/invite list
-                }
-            }
-            else {
-                console.log("Discarding: " + JSON.stringify(data.chunk[i]));
-            }
-        }
-        
-        if (hasNewLatestMessage) {
-           setRooms(roomInfo);
-        }
-        if (updatedMemberList) {
-            $("#members").empty();
-            for (i=0; i<memberInfo.length; ++i) { 
-                addMember(memberInfo[i]);
-            }
-        }
-        longpollEventStream();
-    }).fail(function(err) {
-        setTimeout(longpollEventStream, 5000);
-    });
-};
-
-// ************** Registration and Login **************
-var onLoggedIn = function(data) {
-    accountInfo = data;
-    longpollEventStream();
-    getCurrentRoomList();
-    $(".roomListDashboard").css({visibility: "visible"});
-    $(".roomContents").css({visibility: "visible"});
-    $(".signUp").css({display: "none"});
-};
-
-$('.login').live('click', function() {
-    var user = $("#userLogin").val();
-    var password = $("#passwordLogin").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/login",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            onLoggedIn(data);
-        },
-        error: function(err) {
-            alert("Unable to login: is the home server running?");  
-        }
-    }); 
-});
-
-$('.register').live('click', function() {
-    var user = $("#userReg").val();
-    var password = $("#passwordReg").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/register",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            onLoggedIn(data);
-        },
-        error: function(err) {
-            var msg = "Is the home server running?";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson !== null) {
-                msg = errJson.error;   
-            }
-            alert("Unable to register: "+msg);  
-        }
-    });
-});
-
-// ************** Creating a room ******************
-$('.createRoom').live('click', function() {
-    var roomAlias = $("#roomAlias").val();
-    var data = {};
-    if (roomAlias.length > 0) {
-        data.room_alias_name = roomAlias;   
-    }
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token="+accountInfo.access_token,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(response) {
-            $("#roomAlias").val("");
-            response.membership = "join"; // you are automatically joined into every room you make.
-            response.latest_message = "";
-            
-            roomInfo.push(response);
-            setRooms(roomInfo);
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});
-
-// ************** Getting current state **************
-var getCurrentRoomList = function() {
-    var url = "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=" + accountInfo.access_token + "&limit=1";
-    $.getJSON(url, function(data) {
-        var rooms = data.rooms;
-        for (var i=0; i<rooms.length; ++i) {
-            if ("messages" in rooms[i]) {
-                rooms[i].latest_message = rooms[i].messages.chunk[0].content.body;   
-            }
-        }
-        roomInfo = rooms;
-        setRooms(roomInfo);  
-    }).fail(function(err) {
-        alert(JSON.stringify($.parseJSON(err.responseText)));
-    });
-};
-
-var loadRoomContent = function(roomId) {
-    console.log("loadRoomContent " + roomId);
-    viewingRoomId = roomId;
-    $("#roomName").text("Room: "+roomId);
-    $(".sendMessageForm").css({visibility: "visible"});
-    getMessages(roomId);
-    getMemberList(roomId);
-};
-
-var getMessages = function(roomId) {
-    $("#messages").empty();
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/" + 
-              encodeURIComponent(roomId) + "/messages?access_token=" + accountInfo.access_token + "&from=END&dir=b&limit=10";
-    $.getJSON(url, function(data) {
-        for (var i=data.chunk.length-1; i>=0; --i) {
-            addMessage(data.chunk[i]);   
-        }
-    });
-};
-
-var getMemberList = function(roomId) {
-    $("#members").empty();
-    memberInfo = [];
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/" + 
-              encodeURIComponent(roomId) + "/members?access_token=" + accountInfo.access_token;
-    $.getJSON(url, function(data) {
-        for (var i=0; i<data.chunk.length; ++i) {
-            memberInfo.push(data.chunk[i]);
-            addMember(data.chunk[i]);   
-        }
-    });
-};
-
-// ************** Sending messages **************
-$('.sendMessage').live('click', function() {
-    if (viewingRoomId === undefined) {
-        alert("There is no room to send a message to!");
-        return;
-    }
-    var body = $("#body").val();
-    sendMessage(viewingRoomId, body);
-});
-
-var sendMessage = function(roomId, body) {
-    var msgId = $.now();
-    
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/$roomid/send/m.room.message?access_token=$token";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$roomid", encodeURIComponent(roomId));
-    
-    var data = {
-        msgtype: "m.text",
-        body: body
-    };
-    
-    $.ajax({
-        url: url,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            $("#body").val("");
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    });
-};
-
-// ************** Navigation and DOM manipulation **************
-var setRooms = function(roomList) {
-    // wipe existing entries
-    $("#rooms").find("tr:gt(0)").remove();
-    
-    var rows = "";
-    for (var i=0; i<roomList.length; ++i) {
-        row = "<tr>" +
-              "<td>"+roomList[i].room_id+"</td>" +
-              "<td>"+roomList[i].membership+"</td>" +
-              "<td>"+roomList[i].latest_message+"</td>" +
-              "</tr>";  
-        rows += row;
-    }
-    
-    $("#rooms").append(rows);
-    
-    $('#rooms').find("tr").click(function(){
-        var roomId = $(this).find('td:eq(0)').text();
-        var membership = $(this).find('td:eq(1)').text();
-        if (membership !== "join") {
-            console.log("Joining room " + roomId); 
-            var url = "http://localhost:8008/_matrix/client/api/v1/rooms/$roomid/join?access_token=$token";
-            url = url.replace("$token", accountInfo.access_token);
-            url = url.replace("$roomid", encodeURIComponent(roomId));
-            $.ajax({
-                url: url,
-                type: "POST",
-                contentType: "application/json; charset=utf-8",
-                data: JSON.stringify({membership: "join"}),
-                dataType: "json",
-                success: function(data) {
-                    loadRoomContent(roomId);
-                    getCurrentRoomList();
-                },
-                error: function(err) {
-                    alert(JSON.stringify($.parseJSON(err.responseText)));  
-                }
-            });
-        }
-        else {
-            loadRoomContent(roomId);
-        }
-    });
-};
-
-var addMessage = function(data) {
-
-    var msg = data.content.body;
-    if (data.type === "m.room.member") {
-        if (data.content.membership === undefined) {
-            return;
-        }
-        if (data.content.membership === "invite") {
-            msg = "<em>invited " + data.state_key + " to the room</em>";
-        }
-        else if (data.content.membership === "join") {
-            msg = "<em>joined the room</em>";
-        }
-        else if (data.content.membership === "leave") {
-            msg = "<em>left the room</em>";
-        }
-        else if (data.content.membership === "ban") {
-            msg = "<em>was banned from the room</em>";
-        }
-    }
-    if (msg === undefined) {
-        return;
-    }
-
-    var row = "<tr>" +
-              "<td>"+data.user_id+"</td>" +
-              "<td>"+msg+"</td>" +
-              "</tr>"; 
-    $("#messages").append(row);
-};
-
-var addMember = function(data) {
-    var row = "<tr>" +
-              "<td>"+data.state_key+"</td>" +
-              "<td>"+data.content.membership+"</td>" +
-              "</tr>"; 
-    $("#members").append(row);
-};
-

jsfiddles/register_login/demo.css

@@ -1,7 +0,0 @@
-.loggedin {
-    visibility: hidden;
-}
-
-p {
-    font-family: monospace;
-}

jsfiddles/register_login/demo.html

@@ -1,20 +0,0 @@
-<div>
-    <p>This registration/login demo requires a home server to be running on http://localhost:8008</p>
-</div>
-<form class="registrationForm">
-    <input type="text" id="user" placeholder="Username"></input>
-    <input type="password" id="password" placeholder="Password"></input>
-    <input type="button" class="register" value="Register"></input>
-</form>
-<form class="loginForm">
-    <input type="text" id="userLogin" placeholder="Username"></input>
-    <input type="password" id="passwordLogin" placeholder="Password"></input>
-    <input type="button" class="login" value="Login"></input>
-</form>
-<div class="loggedin">
-    <p id="welcomeText"></p>
-    <input type="button" class="testToken" value="Test token"></input>
-    <input type="button" class="logout" value="Logout"></input>
-    <p id="imSyncText"></p>
-</div>
-    

jsfiddles/register_login/demo.js

@@ -1,79 +0,0 @@
-var accountInfo = {};
-
-var showLoggedIn = function(data) {
-    accountInfo = data;
-    $(".loggedin").css({visibility: "visible"});
-    $("#welcomeText").text("Welcome " + accountInfo.user_id+". Your access token is:  " +
-                           accountInfo.access_token);    
-};
-
-$('.register').live('click', function() {
-    var user = $("#user").val();
-    var password = $("#password").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/register",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            showLoggedIn(data);
-        },
-        error: function(err) {
-            var errMsg = "To try this, you need a home server running!";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson) {
-                errMsg = JSON.stringify(errJson);   
-            }
-            alert(errMsg);   
-        }
-    });
-});
-
-var login = function(user, password) {
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/login",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            showLoggedIn(data);
-        },
-        error: function(err) {
-            var errMsg = "To try this, you need a home server running!";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson) {
-                errMsg = JSON.stringify(errJson);   
-            }
-            alert(errMsg);  
-        }
-    });  
-};
-
-$('.login').live('click', function() {
-    var user = $("#userLogin").val();
-    var password = $("#passwordLogin").val();
-    $.getJSON("http://localhost:8008/_matrix/client/api/v1/login", function(data) {
-        if (data.flows[0].type !== "m.login.password") {
-            alert("I don't know how to login with this type: " + data.type);
-            return;
-        }
-        login(user, password);
-    });
-});
-
-$('.logout').live('click', function() {
-    accountInfo = {};
-    $("#imSyncText").text("");
-    $(".loggedin").css({visibility: "hidden"});
-});
-
-$('.testToken').live('click', function() {
-    var url = "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=" + accountInfo.access_token + "&limit=1";
-    $.getJSON(url, function(data) {
-         $("#imSyncText").text(JSON.stringify(data, undefined, 2));
-    }).fail(function(err) {
-        $("#imSyncText").text(JSON.stringify($.parseJSON(err.responseText)));
-    });
-});

jsfiddles/room_memberships/demo.css

@@ -1,17 +0,0 @@
-.loggedin {
-    visibility: hidden;
-}
-
-p {
-    font-family: monospace;
-}
-
-table
-{
-    border-spacing:5px;
-}
-
-th,td
-{
-    padding:5px;
-}

jsfiddles/room_memberships/demo.html

@@ -1,37 +0,0 @@
-<div>
-    <p>This room membership demo requires a home server to be running on http://localhost:8008</p>
-</div>
-<form class="loginForm">
-    <input type="text" id="userLogin" placeholder="Username"></input>
-    <input type="password" id="passwordLogin" placeholder="Password"></input>
-    <input type="button" class="login" value="Login"></input>
-</form>
-<div class="loggedin">
-    <form class="createRoomForm">
-        <input type="button" class="createRoom" value="Create Room"></input>
-    </form>
-    <form class="changeMembershipForm">
-        <input type="text" id="roomId" placeholder="Room ID"></input>
-        <input type="text" id="targetUser" placeholder="Target User ID"></input>
-        <select id="membership">
-            <option value="invite">invite</option>
-            <option value="join">join</option>
-            <option value="leave">leave</option>
-        </select>
-        <input type="button" class="changeMembership" value="Change Membership"></input>
-    </form>
-    <form class="joinAliasForm">
-        <input type="text" id="roomAlias" placeholder="Room Alias (#name:domain)"></input>
-        <input type="button" class="joinAlias" value="Join via Alias"></input>
-    </form>
-    <table id="rooms">
-        <tbody>
-            <tr>
-                <th>Room ID</th>
-                <th>My state</th>
-                <th>Room Alias</th>
-            </tr>
-        </tbody>
-    </table>
-</div>
-    

jsfiddles/room_memberships/demo.js

@@ -1,141 +0,0 @@
-var accountInfo = {};
-
-var showLoggedIn = function(data) {
-    accountInfo = data;
-    getCurrentRoomList();
-    $(".loggedin").css({visibility: "visible"});
-    $("#membership").change(function() {
-    if ($("#membership").val() === "invite") {
-        $("#targetUser").css({visibility: "visible"});
-    }
-    else {
-        $("#targetUser").css({visibility: "hidden"});
-    }
-});
-};
-
-$('.login').live('click', function() {
-    var user = $("#userLogin").val();
-    var password = $("#passwordLogin").val();
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/login",
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({ user: user, password: password, type: "m.login.password" }),
-        dataType: "json",
-        success: function(data) {
-            $("#rooms").find("tr:gt(0)").remove();
-            showLoggedIn(data);
-        },
-        error: function(err) {
-            var errMsg = "To try this, you need a home server running!";
-            var errJson = $.parseJSON(err.responseText);
-            if (errJson) {
-                errMsg = JSON.stringify(errJson);   
-            }
-            alert(errMsg); 
-        }
-    }); 
-});
-
-var getCurrentRoomList = function() {
-    $("#roomId").val("");
-    // wipe the table and reload it. Using the event stream would be the best
-    // solution but that is out of scope of this fiddle.
-    $("#rooms").find("tr:gt(0)").remove();
-    
-    var url = "http://localhost:8008/_matrix/client/api/v1/initialSync?access_token=" + accountInfo.access_token + "&limit=1";
-    $.getJSON(url, function(data) {
-        var rooms = data.rooms;
-        for (var i=0; i<rooms.length; ++i) {
-            addRoom(rooms[i]);   
-        }
-    }).fail(function(err) {
-        alert(JSON.stringify($.parseJSON(err.responseText)));
-    });
-};
-
-$('.createRoom').live('click', function() {
-    var data = {};
-    $.ajax({
-        url: "http://localhost:8008/_matrix/client/api/v1/createRoom?access_token="+accountInfo.access_token,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            data.membership = "join"; // you are automatically joined into every room you make.
-            data.latest_message = "";
-            addRoom(data);
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});
-
-var addRoom = function(data) {
-    row = "<tr>" +
-        "<td>"+data.room_id+"</td>" +
-        "<td>"+data.membership+"</td>" +
-        "<td>"+data.room_alias+"</td>" +
-        "</tr>";
-    $("#rooms").append(row);
-};
-
-$('.changeMembership').live('click', function() {
-    var roomId = $("#roomId").val();
-    var member = $("#targetUser").val();
-    var membership = $("#membership").val();
-    
-    if (roomId.length === 0) {
-        return;
-    }
-    
-    var url = "http://localhost:8008/_matrix/client/api/v1/rooms/$roomid/$membership?access_token=$token";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$roomid", encodeURIComponent(roomId));
-    url = url.replace("$membership", membership);
-    
-    var data = {};
-    
-    if (membership === "invite") {
-        data = {
-            user_id: member
-        };
-    }
-
-    $.ajax({
-        url: url,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify(data),
-        dataType: "json",
-        success: function(data) {
-            getCurrentRoomList();
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});
-
-$('.joinAlias').live('click', function() {
-    var roomAlias = $("#roomAlias").val();
-    var url = "http://localhost:8008/_matrix/client/api/v1/join/$roomalias?access_token=$token";
-    url = url.replace("$token", accountInfo.access_token);
-    url = url.replace("$roomalias", encodeURIComponent(roomAlias));
-    $.ajax({
-        url: url,
-        type: "POST",
-        contentType: "application/json; charset=utf-8",
-        data: JSON.stringify({}),
-        dataType: "json",
-        success: function(data) {
-            getCurrentRoomList();
-        },
-        error: function(err) {
-            alert(JSON.stringify($.parseJSON(err.responseText)));  
-        }
-    }); 
-});

scripts/copyrighter-sql.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/perl -pi
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$copyright = <<EOT;
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+EOT
+
+s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1);

scripts/copyrighter.pl

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 $copyright = <<EOT;
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" This is a reference implementation of a synapse home server.
+""" This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.6.0"
+__version__ = "0.6.1b"

synapse/api/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/constants.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/errors.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/urls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/homeserver.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@ from synapse.storage import prepare_database, UpgradeDatabaseException
 
 from synapse.server import HomeServer
 
+from synapse.python_dependencies import check_requirements
+
 from twisted.internet import reactor
 from twisted.enterprise import adbapi
 from twisted.web.resource import Resource
@@ -39,6 +41,8 @@ from synapse.util.logcontext import LoggingContext
 from daemonize import Daemonize
 import twisted.manhole.telnet
 
+import synapse
+
 import logging
 import os
 import re
@@ -198,7 +202,10 @@ def setup():
 
     config.setup_logging()
 
+    check_requirements()
+
     logger.info("Server hostname: %s", config.server_name)
+    logger.info("Server version: %s", synapse.__version__)
 
     if re.search(":[0-9]+$", config.server_name):
         domain_with_port = config.server_name
@@ -234,13 +241,20 @@ def setup():
     except UpgradeDatabaseException:
         sys.stderr.write(
             "\nFailed to upgrade database.\n"
-            "Have you followed any instructions in UPGRADES.rst?\n"
+            "Have you checked for version specific instructions in"
+            " UPGRADES.rst?\n"
         )
         sys.exit(1)
 
     logger.info("Database prepared in %s.", db_name)
 
-    hs.get_db_pool()
+    db_pool = hs.get_db_pool()
+
+    if db_name == ":memory:":
+        # Memory databases will need to be setup each time they are opened.
+        reactor.callWhenRunning(
+            db_pool.runWithConnection, prepare_database
+        )
 
     if config.manhole:
         f = twisted.manhole.telnet.ShellFactory()
@@ -277,6 +291,7 @@ def run():
 
 def main():
     with LoggingContext("main"):
+        check_requirements()
         setup()
 
 

synapse/app/synctl.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/captcha.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,10 @@ import os
 class DatabaseConfig(Config):
     def __init__(self, args):
         super(DatabaseConfig, self).__init__(args)
-        self.database_path = self.abspath(args.database_path)
+        if args.database_path == ":memory:":
+            self.database_path = ":memory:"
+        else:
+            self.database_path = self.abspath(args.database_path)
 
     @classmethod
     def add_arguments(cls, parser):

synapse/config/email.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/homeserver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/logger.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -66,7 +66,10 @@ class LoggingConfig(Config):
 
             formatter = logging.Formatter(log_format)
             if self.log_file:
-                handler = logging.FileHandler(self.log_file)
+                # TODO: Customisable file size / backup count
+                handler = logging.handlers.RotatingFileHandler(
+                    self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3
+                )
             else:
                 handler = logging.StreamHandler()
             handler.setFormatter(formatter)

synapse/config/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014, 2015 matrix.org
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config):
         super(ContentRepositoryConfig, cls).add_arguments(parser)
         db_group = parser.add_argument_group("content_repository")
         db_group.add_argument(
-            "--max-upload-size", default="1M"
+            "--max-upload-size", default="10M"
         )
         db_group.add_argument(
             "--media-store-path", default=cls.default_path("media_store")

synapse/config/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,8 +47,12 @@ class ServerConfig(Config):
     def add_arguments(cls, parser):
         super(ServerConfig, cls).add_arguments(parser)
         server_group = parser.add_argument_group("server")
-        server_group.add_argument("-H", "--server-name", default="localhost",
-                                  help="The name of the server")
+        server_group.add_argument(
+            "-H", "--server-name", default="localhost",
+            help="The domain name of the server, with optional explicit port. "
+                 "This is used by remote servers to connect to this server, "
+                 "e.g. matrix.org, localhost:8080, etc."
+        )
         server_group.add_argument("--signing-key-path",
                                   help="The signing key to sign messages with")
         server_group.add_argument("-p", "--bind-port", metavar="PORT",

synapse/config/tls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/voip.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/context_factory.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/event_signing.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyclient.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyring.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/events/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +15,10 @@
 
 from synapse.util.frozenutils import freeze, unfreeze
 
-import copy
-
 
 class _EventInternalMetadata(object):
     def __init__(self, internal_metadata_dict):
-        self.__dict__ = copy.deepcopy(internal_metadata_dict)
+        self.__dict__ = internal_metadata_dict
 
     def get_dict(self):
         return dict(self.__dict__)
@@ -49,10 +47,10 @@ def _event_dict_property(key):
 class EventBase(object):
     def __init__(self, event_dict, signatures={}, unsigned={},
                  internal_metadata_dict={}):
-        self.signatures = copy.deepcopy(signatures)
-        self.unsigned = copy.deepcopy(unsigned)
+        self.signatures = signatures
+        self.unsigned = unsigned
 
-        self._event_dict = copy.deepcopy(event_dict)
+        self._event_dict = event_dict
 
         self.internal_metadata = _EventInternalMetadata(
             internal_metadata_dict
@@ -112,10 +110,16 @@ class EventBase(object):
 
 class FrozenEvent(EventBase):
     def __init__(self, event_dict, internal_metadata_dict={}):
-        event_dict = copy.deepcopy(event_dict)
+        event_dict = dict(event_dict)
 
-        signatures = copy.deepcopy(event_dict.pop("signatures", {}))
-        unsigned = copy.deepcopy(event_dict.pop("unsigned", {}))
+        # Signatures is a dict of dicts, and this is faster than doing a
+        # copy.deepcopy
+        signatures = {
+            name: {sig_id: sig for sig_id, sig in sigs.items()}
+            for name, sigs in event_dict.pop("signatures", {}).items()
+        }
+
+        unsigned = dict(event_dict.pop("unsigned", {}))
 
         frozen_dict = freeze(event_dict)
 

synapse/events/builder.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,12 +33,6 @@ class EventBuilder(EventBase):
             unsigned=unsigned
         )
 
-    def update_event_key(self, key, value):
-        self._event_dict[key] = value
-
-    def update_event_keys(self, other_dict):
-        self._event_dict.update(other_dict)
-
     def build(self):
         return FrozenEvent.from_event(self)
 

synapse/events/snapshot.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/events/utils.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -89,16 +89,24 @@ def prune_event(event):
     return type(event)(allowed_fields)
 
 
-def serialize_event(hs, e):
+def serialize_event(hs, e, client_event=True):
     # FIXME(erikj): To handle the case of presence events and the like
     if not isinstance(e, EventBase):
         return e
 
     # Should this strip out None's?
     d = {k: v for k, v in e.get_dict().items()}
+
+    if not client_event:
+        # set the age and keep all other keys
+        if "age_ts" in d["unsigned"]:
+            now = int(hs.get_clock().time_msec())
+            d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
+        return d
+
     if "age_ts" in d["unsigned"]:
         now = int(hs.get_clock().time_msec())
-        d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
+        d["age"] = now - d["unsigned"]["age_ts"]
         del d["unsigned"]["age_ts"]
 
     d["user_id"] = d.pop("sender", None)

synapse/events/validator.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/persistence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/replication.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -256,23 +256,21 @@ class ReplicationLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def get_state_for_context(self, destination, context, event_id):
-        """Requests all of the `current` state PDUs for a given context from
+    def get_state_for_room(self, destination, room_id, event_id):
+        """Requests all of the `current` state PDUs for a given room from
         a remote home server.
 
         Args:
             destination (str): The remote homeserver to query for the state.
-            context (str): The context we're interested in.
+            room_id (str): The id of the room we're interested in.
             event_id (str): The id of the event we want the state at.
 
         Returns:
             Deferred: Results in a list of PDUs.
         """
 
-        result = yield self.transport_layer.get_context_state(
-            destination,
-            context,
-            event_id=event_id,
+        result = yield self.transport_layer.get_room_state(
+            destination, room_id, event_id=event_id,
         )
 
         pdus = [
@@ -288,9 +286,9 @@ class ReplicationLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def get_event_auth(self, destination, context, event_id):
+    def get_event_auth(self, destination, room_id, event_id):
         res = yield self.transport_layer.get_event_auth(
-            destination, context, event_id,
+            destination, room_id, event_id,
         )
 
         auth_chain = [
@@ -304,9 +302,9 @@ class ReplicationLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def on_backfill_request(self, origin, context, versions, limit):
+    def on_backfill_request(self, origin, room_id, versions, limit):
         pdus = yield self.handler.on_backfill_request(
-            origin, context, versions, limit
+            origin, room_id, versions, limit
         )
 
         defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict()))
@@ -380,12 +378,10 @@ class ReplicationLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def on_context_state_request(self, origin, context, event_id):
+    def on_context_state_request(self, origin, room_id, event_id):
         if event_id:
             pdus = yield self.handler.get_state_for_pdu(
-                origin,
-                context,
-                event_id,
+                origin, room_id, event_id,
             )
             auth_chain = yield self.store.get_auth_chain(
                 [pdu.event_id for pdu in pdus]
@@ -413,7 +409,7 @@ class ReplicationLayer(object):
     @defer.inlineCallbacks
     @log_function
     def on_pull_request(self, origin, versions):
-        raise NotImplementedError("Pull transacions not implemented")
+        raise NotImplementedError("Pull transactions not implemented")
 
     @defer.inlineCallbacks
     def on_query_request(self, query_type, args):
@@ -422,30 +418,21 @@ class ReplicationLayer(object):
             defer.returnValue((200, response))
         else:
             defer.returnValue(
-                (404, "No handler for Query type '%s'" % (query_type, ))
+                (404, "No handler for Query type '%s'" % (query_type,))
             )
 
     @defer.inlineCallbacks
-    def on_make_join_request(self, context, user_id):
-        pdu = yield self.handler.on_make_join_request(context, user_id)
+    def on_make_join_request(self, room_id, user_id):
+        pdu = yield self.handler.on_make_join_request(room_id, user_id)
         time_now = self._clock.time_msec()
-        defer.returnValue({
-            "event": pdu.get_pdu_json(time_now),
-        })
+        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content):
         pdu = self.event_from_pdu_json(content)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
         time_now = self._clock.time_msec()
-        defer.returnValue(
-            (
-                200,
-                {
-                    "event": ret_pdu.get_pdu_json(time_now),
-                }
-            )
-        )
+        defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)}))
 
     @defer.inlineCallbacks
     def on_send_join_request(self, origin, content):
@@ -462,26 +449,17 @@ class ReplicationLayer(object):
         }))
 
     @defer.inlineCallbacks
-    def on_event_auth(self, origin, context, event_id):
+    def on_event_auth(self, origin, room_id, event_id):
         time_now = self._clock.time_msec()
         auth_pdus = yield self.handler.on_event_auth(event_id)
-        defer.returnValue(
-            (
-                200,
-                {
-                    "auth_chain": [
-                        a.get_pdu_json(time_now) for a in auth_pdus
-                    ],
-                }
-            )
-        )
+        defer.returnValue((200, {
+            "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus],
+        }))
 
     @defer.inlineCallbacks
-    def make_join(self, destination, context, user_id):
+    def make_join(self, destination, room_id, user_id):
         ret = yield self.transport_layer.make_join(
-            destination=destination,
-            context=context,
-            user_id=user_id,
+            destination, room_id, user_id
         )
 
         pdu_dict = ret["event"]
@@ -494,10 +472,10 @@ class ReplicationLayer(object):
     def send_join(self, destination, pdu):
         time_now = self._clock.time_msec()
         _, content = yield self.transport_layer.send_join(
-            destination,
-            pdu.room_id,
-            pdu.event_id,
-            pdu.get_pdu_json(time_now),
+            destination=destination,
+            room_id=pdu.room_id,
+            event_id=pdu.event_id,
+            content=pdu.get_pdu_json(time_now),
         )
 
         logger.debug("Got content: %s", content)
@@ -507,9 +485,6 @@ class ReplicationLayer(object):
             for p in content.get("state", [])
         ]
 
-        # FIXME: We probably want to do something with the auth_chain given
-        # to us
-
         auth_chain = [
             self.event_from_pdu_json(p, outlier=True)
             for p in content.get("auth_chain", [])
@@ -523,11 +498,11 @@ class ReplicationLayer(object):
         })
 
     @defer.inlineCallbacks
-    def send_invite(self, destination, context, event_id, pdu):
+    def send_invite(self, destination, room_id, event_id, pdu):
         time_now = self._clock.time_msec()
         code, content = yield self.transport_layer.send_invite(
             destination=destination,
-            context=context,
+            room_id=room_id,
             event_id=event_id,
             content=pdu.get_pdu_json(time_now),
         )
@@ -657,7 +632,7 @@ class ReplicationLayer(object):
                     "_handle_new_pdu getting state for %s",
                     pdu.room_id
                 )
-                state, auth_chain = yield self.get_state_for_context(
+                state, auth_chain = yield self.get_state_for_room(
                     origin, pdu.room_id, pdu.event_id,
                 )
 
@@ -730,6 +705,7 @@ class _TransactionQueue(object):
 
         destinations = set(destinations)
         destinations.discard(self.server_name)
+        destinations.discard("localhost")
 
         logger.debug("Sending to: %s", str(destinations))
 
@@ -814,6 +790,8 @@ class _TransactionQueue(object):
             else:
                 logger.info("TX [%s] is ready for retry", destination)
 
+        logger.info("TX [%s] _attempt_new_transaction", destination)
+
         if destination in self.pending_transactions:
             # XXX: pending_transactions can get stuck on by a never-ending
             # request at which point pending_pdus_by_dest just keeps growing.
@@ -826,12 +804,16 @@ class _TransactionQueue(object):
         pending_edus = self.pending_edus_by_dest.pop(destination, [])
         pending_failures = self.pending_failures_by_dest.pop(destination, [])
 
+        if pending_pdus:
+            logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d",
+                        destination, len(pending_pdus))
+
         if not pending_pdus and not pending_edus and not pending_failures:
             return
 
         logger.debug(
-            "TX [%s] Attempting new transaction "
-            "(pdus: %d, edus: %d, failures: %d)",
+            "TX [%s] Attempting new transaction"
+            " (pdus: %d, edus: %d, failures: %d)",
             destination,
             len(pending_pdus),
             len(pending_edus),

synapse/federation/transport/__init__.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The transport layer is responsible for both sending transactions to remote
+home servers and receiving a variety of requests from other home servers.
+
+By default this is done over HTTPS (and all home servers are required to
+support HTTPS), however individual pairings of servers may decide to
+communicate over a different (albeit still reliable) protocol.
+"""
+
+from .server import TransportLayerServer
+from .client import TransportLayerClient
+
+
+class TransportLayer(TransportLayerServer, TransportLayerClient):
+    """This is a basic implementation of the transport layer that translates
+    transactions and other requests to/from HTTP.
+
+    Attributes:
+        server_name (str): Local home server host
+
+        server (synapse.http.server.HttpServer): the http server to
+                register listeners on
+
+        client (synapse.http.client.HttpClient): the http client used to
+                send requests
+
+        request_handler (TransportRequestHandler): The handler to fire when we
+            receive requests for data.
+
+        received_handler (TransportReceivedHandler): The handler to fire when
+            we receive data.
+    """
+
+    def __init__(self, homeserver, server_name, server, client):
+        """
+        Args:
+            server_name (str): Local home server host
+            server (synapse.protocol.http.HttpServer): the http server to
+                register listeners on
+            client (synapse.protocol.http.HttpClient): the http client used to
+                send requests
+        """
+        self.keyring = homeserver.get_keyring()
+        self.server_name = server_name
+        self.server = server
+        self.client = client
+        self.request_handler = None
+        self.received_handler = None

synapse/federation/transport/client.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.urls import FEDERATION_PREFIX as PREFIX
+from synapse.util.logutils import log_function
+
+import logging
+import json
+
+
+logger = logging.getLogger(__name__)
+
+
+class TransportLayerClient(object):
+    """Sends federation HTTP requests to other servers"""
+
+    @log_function
+    def get_room_state(self, destination, room_id, event_id):
+        """ Requests all state for a given room from the given server at the
+        given event.
+
+        Args:
+            destination (str): The host name of the remote home server we want
+                to get the state from.
+            context (str): The name of the context we want the state of
+            event_id (str): The event we want the context at.
+
+        Returns:
+            Deferred: Results in a dict received from the remote homeserver.
+        """
+        logger.debug("get_room_state dest=%s, room=%s",
+                     destination, room_id)
+
+        path = PREFIX + "/state/%s/" % room_id
+        return self.client.get_json(
+            destination, path=path, args={"event_id": event_id},
+        )
+
+    @log_function
+    def get_event(self, destination, event_id):
+        """ Requests the pdu with give id and origin from the given server.
+
+        Args:
+            destination (str): The host name of the remote home server we want
+                to get the state from.
+            event_id (str): The id of the event being requested.
+
+        Returns:
+            Deferred: Results in a dict received from the remote homeserver.
+        """
+        logger.debug("get_pdu dest=%s, event_id=%s",
+                     destination, event_id)
+
+        path = PREFIX + "/event/%s/" % (event_id, )
+        return self.client.get_json(destination, path=path)
+
+    @log_function
+    def backfill(self, destination, room_id, event_tuples, limit):
+        """ Requests `limit` previous PDUs in a given context before list of
+        PDUs.
+
+        Args:
+            dest (str)
+            room_id (str)
+            event_tuples (list)
+            limt (int)
+
+        Returns:
+            Deferred: Results in a dict received from the remote homeserver.
+        """
+        logger.debug(
+            "backfill dest=%s, room_id=%s, event_tuples=%s, limit=%s",
+            destination, room_id, repr(event_tuples), str(limit)
+        )
+
+        if not event_tuples:
+            # TODO: raise?
+            return
+
+        path = PREFIX + "/backfill/%s/" % (room_id,)
+
+        args = {
+            "v": event_tuples,
+            "limit": [str(limit)],
+        }
+
+        return self.client.get_json(
+            destination,
+            path=path,
+            args=args,
+        )
+
+    @defer.inlineCallbacks
+    @log_function
+    def send_transaction(self, transaction, json_data_callback=None):
+        """ Sends the given Transaction to its destination
+
+        Args:
+            transaction (Transaction)
+
+        Returns:
+            Deferred: Results of the deferred is a tuple in the form of
+            (response_code, response_body) where the response_body is a
+            python dict decoded from json
+        """
+        logger.debug(
+            "send_data dest=%s, txid=%s",
+            transaction.destination, transaction.transaction_id
+        )
+
+        if transaction.destination == self.server_name:
+            raise RuntimeError("Transport layer cannot send to itself!")
+
+        # FIXME: This is only used by the tests. The actual json sent is
+        # generated by the json_data_callback.
+        json_data = transaction.get_dict()
+
+        code, response = yield self.client.put_json(
+            transaction.destination,
+            path=PREFIX + "/send/%s/" % transaction.transaction_id,
+            data=json_data,
+            json_data_callback=json_data_callback,
+        )
+
+        logger.debug(
+            "send_data dest=%s, txid=%s, got response: %d",
+            transaction.destination, transaction.transaction_id, code
+        )
+
+        defer.returnValue((code, response))
+
+    @defer.inlineCallbacks
+    @log_function
+    def make_query(self, destination, query_type, args, retry_on_dns_fail):
+        path = PREFIX + "/query/%s" % query_type
+
+        response = yield self.client.get_json(
+            destination=destination,
+            path=path,
+            args=args,
+            retry_on_dns_fail=retry_on_dns_fail,
+        )
+
+        defer.returnValue(response)
+
+    @defer.inlineCallbacks
+    @log_function
+    def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True):
+        path = PREFIX + "/make_join/%s/%s" % (room_id, user_id)
+
+        response = yield self.client.get_json(
+            destination=destination,
+            path=path,
+            retry_on_dns_fail=retry_on_dns_fail,
+        )
+
+        defer.returnValue(response)
+
+    @defer.inlineCallbacks
+    @log_function
+    def send_join(self, destination, room_id, event_id, content):
+        path = PREFIX + "/send_join/%s/%s" % (room_id, event_id)
+
+        code, content = yield self.client.put_json(
+            destination=destination,
+            path=path,
+            data=content,
+        )
+
+        if not 200 <= code < 300:
+            raise RuntimeError("Got %d from send_join", code)
+
+        defer.returnValue(json.loads(content))
+
+    @defer.inlineCallbacks
+    @log_function
+    def send_invite(self, destination, room_id, event_id, content):
+        path = PREFIX + "/invite/%s/%s" % (room_id, event_id)
+
+        code, content = yield self.client.put_json(
+            destination=destination,
+            path=path,
+            data=content,
+        )
+
+        if not 200 <= code < 300:
+            raise RuntimeError("Got %d from send_invite", code)
+
+        defer.returnValue(json.loads(content))
+
+    @defer.inlineCallbacks
+    @log_function
+    def get_event_auth(self, destination, room_id, event_id):
+        path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id)
+
+        response = yield self.client.get_json(
+            destination=destination,
+            path=path,
+        )
+
+        defer.returnValue(response)

synapse/federation/transport/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""The transport layer is responsible for both sending transactions to remote
-home servers and receiving a variety of requests from other home servers.
-
-Typically, this is done over HTTP (and all home servers are required to
-support HTTP), however individual pairings of servers may decide to communicate
-over a different (albeit still reliable) protocol.
-"""
-
 from twisted.internet import defer
 
 from synapse.api.urls import FEDERATION_PREFIX as PREFIX
@@ -35,241 +27,8 @@ import re
 logger = logging.getLogger(__name__)
 
 
-class TransportLayer(object):
-    """This is a basic implementation of the transport layer that translates
-    transactions and other requests to/from HTTP.
-
-    Attributes:
-        server_name (str): Local home server host
-
-        server (synapse.http.server.HttpServer): the http server to
-                register listeners on
-
-        client (synapse.http.client.HttpClient): the http client used to
-                send requests
-
-        request_handler (TransportRequestHandler): The handler to fire when we
-            receive requests for data.
-
-        received_handler (TransportReceivedHandler): The handler to fire when
-            we receive data.
-    """
-
-    def __init__(self, homeserver, server_name, server, client):
-        """
-        Args:
-            server_name (str): Local home server host
-            server (synapse.protocol.http.HttpServer): the http server to
-                register listeners on
-            client (synapse.protocol.http.HttpClient): the http client used to
-                send requests
-        """
-        self.keyring = homeserver.get_keyring()
-        self.server_name = server_name
-        self.server = server
-        self.client = client
-        self.request_handler = None
-        self.received_handler = None
-
-    @log_function
-    def get_context_state(self, destination, context, event_id=None):
-        """ Requests all state for a given context (i.e. room) from the
-        given server.
-
-        Args:
-            destination (str): The host name of the remote home server we want
-                to get the state from.
-            context (str): The name of the context we want the state of
-
-        Returns:
-            Deferred: Results in a dict received from the remote homeserver.
-        """
-        logger.debug("get_context_state dest=%s, context=%s",
-                     destination, context)
-
-        subpath = "/state/%s/" % context
-
-        args = {}
-        if event_id:
-            args["event_id"] = event_id
-
-        return self._do_request_for_transaction(
-            destination, subpath, args=args
-        )
-
-    @log_function
-    def get_event(self, destination, event_id):
-        """ Requests the pdu with give id and origin from the given server.
-
-        Args:
-            destination (str): The host name of the remote home server we want
-                to get the state from.
-            event_id (str): The id of the event being requested.
-
-        Returns:
-            Deferred: Results in a dict received from the remote homeserver.
-        """
-        logger.debug("get_pdu dest=%s, event_id=%s",
-                     destination, event_id)
-
-        subpath = "/event/%s/" % (event_id, )
-
-        return self._do_request_for_transaction(destination, subpath)
-
-    @log_function
-    def backfill(self, dest, context, event_tuples, limit):
-        """ Requests `limit` previous PDUs in a given context before list of
-        PDUs.
-
-        Args:
-            dest (str)
-            context (str)
-            event_tuples (list)
-            limt (int)
-
-        Returns:
-            Deferred: Results in a dict received from the remote homeserver.
-        """
-        logger.debug(
-            "backfill dest=%s, context=%s, event_tuples=%s, limit=%s",
-            dest, context, repr(event_tuples), str(limit)
-        )
-
-        if not event_tuples:
-            # TODO: raise?
-            return
-
-        subpath = "/backfill/%s/" % (context,)
-
-        args = {
-            "v": event_tuples,
-            "limit": [str(limit)],
-        }
-
-        return self._do_request_for_transaction(
-            dest,
-            subpath,
-            args=args,
-        )
-
-    @defer.inlineCallbacks
-    @log_function
-    def send_transaction(self, transaction, json_data_callback=None):
-        """ Sends the given Transaction to its destination
-
-        Args:
-            transaction (Transaction)
-
-        Returns:
-            Deferred: Results of the deferred is a tuple in the form of
-            (response_code, response_body) where the response_body is a
-            python dict decoded from json
-        """
-        logger.debug(
-            "send_data dest=%s, txid=%s",
-            transaction.destination, transaction.transaction_id
-        )
-
-        if transaction.destination == self.server_name:
-            raise RuntimeError("Transport layer cannot send to itself!")
-
-        # FIXME: This is only used by the tests. The actual json sent is
-        # generated by the json_data_callback.
-        json_data = transaction.get_dict()
-
-        code, response = yield self.client.put_json(
-            transaction.destination,
-            path=PREFIX + "/send/%s/" % transaction.transaction_id,
-            data=json_data,
-            json_data_callback=json_data_callback,
-        )
-
-        logger.debug(
-            "send_data dest=%s, txid=%s, got response: %d",
-            transaction.destination, transaction.transaction_id, code
-        )
-
-        defer.returnValue((code, response))
-
-    @defer.inlineCallbacks
-    @log_function
-    def make_query(self, destination, query_type, args, retry_on_dns_fail):
-        path = PREFIX + "/query/%s" % query_type
-
-        response = yield self.client.get_json(
-            destination=destination,
-            path=path,
-            args=args,
-            retry_on_dns_fail=retry_on_dns_fail,
-        )
-
-        defer.returnValue(response)
-
-    @defer.inlineCallbacks
-    @log_function
-    def make_join(self, destination, context, user_id, retry_on_dns_fail=True):
-        path = PREFIX + "/make_join/%s/%s" % (context, user_id,)
-
-        response = yield self.client.get_json(
-            destination=destination,
-            path=path,
-            retry_on_dns_fail=retry_on_dns_fail,
-        )
-
-        defer.returnValue(response)
-
-    @defer.inlineCallbacks
-    @log_function
-    def send_join(self, destination, context, event_id, content):
-        path = PREFIX + "/send_join/%s/%s" % (
-            context,
-            event_id,
-        )
-
-        code, content = yield self.client.put_json(
-            destination=destination,
-            path=path,
-            data=content,
-        )
-
-        if not 200 <= code < 300:
-            raise RuntimeError("Got %d from send_join", code)
-
-        defer.returnValue(json.loads(content))
-
-    @defer.inlineCallbacks
-    @log_function
-    def send_invite(self, destination, context, event_id, content):
-        path = PREFIX + "/invite/%s/%s" % (
-            context,
-            event_id,
-        )
-
-        code, content = yield self.client.put_json(
-            destination=destination,
-            path=path,
-            data=content,
-        )
-
-        if not 200 <= code < 300:
-            raise RuntimeError("Got %d from send_invite", code)
-
-        defer.returnValue(json.loads(content))
-
-    @defer.inlineCallbacks
-    @log_function
-    def get_event_auth(self, destination, context, event_id):
-        path = PREFIX + "/event_auth/%s/%s" % (
-            context,
-            event_id,
-        )
-
-        response = yield self.client.get_json(
-            destination=destination,
-            path=path,
-        )
-
-        defer.returnValue(response)
+class TransportLayerServer(object):
+    """Handles incoming federation HTTP requests"""
 
     @defer.inlineCallbacks
     def _authenticate_request(self, request):
@@ -373,8 +132,6 @@ class TransportLayer(object):
         """
         self.request_handler = handler
 
-        # TODO(markjh): Namespace the federation URI paths
-
         # This is for when someone asks us for everything since version X
         self.server.register_path(
             "GET",
@@ -528,34 +285,6 @@ class TransportLayer(object):
 
         defer.returnValue((code, response))
 
-    @defer.inlineCallbacks
-    @log_function
-    def _do_request_for_transaction(self, destination, subpath, args={}):
-        """
-        Args:
-            destination (str)
-            path (str)
-            args (dict): This is parsed directly to the HttpClient.
-
-        Returns:
-            Deferred: Results in a dict.
-        """
-
-        data = yield self.client.get_json(
-            destination,
-            path=PREFIX + subpath,
-            args=args,
-        )
-
-        # Add certain keys to the JSON, ready for decoding as a Transaction
-        data.update(
-            origin=destination,
-            destination=self.server_name,
-            transaction_id=None
-        )
-
-        defer.returnValue(data)
-
     @log_function
     def _on_backfill_request(self, origin, context, v_list, limits):
         if not limits:

synapse/federation/units.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -144,7 +144,5 @@ class BaseHandler(object):
         yield self.notifier.on_new_room_event(event, extra_users=extra_users)
 
         yield federation_handler.handle_new_event(
-            event,
-            None,
-            destinations=destinations,
+            event, destinations=destinations,
         )

synapse/handlers/admin.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -155,4 +155,4 @@ class DirectoryHandler(BaseHandler):
             "room_id": room_id,
             "sender": user_id,
             "content": {"aliases": aliases},
-        })
+        }, ratelimit=False)

synapse/handlers/events.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,7 +46,8 @@ class EventStreamHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def get_stream(self, auth_user_id, pagin_config, timeout=0):
+    def get_stream(self, auth_user_id, pagin_config, timeout=0,
+                   as_client_event=True):
         auth_user = self.hs.parse_userid(auth_user_id)
 
         try:
@@ -69,16 +70,16 @@ class EventStreamHandler(BaseHandler):
                 pagin_config.from_token = None
 
             rm_handler = self.hs.get_handlers().room_member_handler
-            logger.debug("BETA")
             room_ids = yield rm_handler.get_rooms_for_user(auth_user)
 
-            logger.debug("ALPHA")
             with PreserveLoggingContext():
                 events, tokens = yield self.notifier.get_events_for(
                     auth_user, room_ids, pagin_config, timeout
                 )
 
-            chunks = [self.hs.serialize_event(e) for e in events]
+            chunks = [
+                self.hs.serialize_event(e, as_client_event) for e in events
+            ]
 
             chunk = {
                 "chunk": chunks,

synapse/handlers/federation.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -75,14 +75,14 @@ class FederationHandler(BaseHandler):
 
     @log_function
     @defer.inlineCallbacks
-    def handle_new_event(self, event, snapshot, destinations):
+    def handle_new_event(self, event, destinations):
         """ Takes in an event from the client to server side, that has already
         been authed and handled by the state module, and sends it to any
         remote home servers that may be interested.
 
         Args:
-            event
-            snapshot (.storage.Snapshot): THe snapshot the event happened after
+            event: The event to send
+            destinations: A list of destinations to send it to
 
         Returns:
             Deferred: Resolved when it has successfully been queued for
@@ -154,7 +154,7 @@ class FederationHandler(BaseHandler):
             replication = self.replication_layer
 
             if not state:
-                state, auth_chain = yield replication.get_state_for_context(
+                state, auth_chain = yield replication.get_state_for_room(
                     origin, context=event.room_id, event_id=event.event_id,
                 )
 
@@ -281,7 +281,7 @@ class FederationHandler(BaseHandler):
         """
         pdu = yield self.replication_layer.send_invite(
             destination=target_host,
-            context=event.room_id,
+            room_id=event.room_id,
             event_id=event.event_id,
             pdu=event
         )
@@ -617,13 +617,13 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def on_backfill_request(self, origin, context, pdu_list, limit):
-        in_room = yield self.auth.check_host_in_room(context, origin)
+    def on_backfill_request(self, origin, room_id, pdu_list, limit):
+        in_room = yield self.auth.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
         events = yield self.store.get_backfill_events(
-            context,
+            room_id,
             pdu_list,
             limit
         )

synapse/handlers/login.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/message.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import RoomError
 from synapse.streams.config import PaginationConfig
 from synapse.events.validator import EventValidator
+from synapse.util.logcontext import PreserveLoggingContext
 
 from ._base import BaseHandler
 
@@ -66,7 +67,7 @@ class MessageHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_messages(self, user_id=None, room_id=None, pagin_config=None,
-                     feedback=False):
+                     feedback=False, as_client_event=True):
         """Get messages in a room.
 
         Args:
@@ -75,6 +76,7 @@ class MessageHandler(BaseHandler):
             pagin_config (synapse.api.streams.PaginationConfig): The pagination
             config rules to apply, if any.
             feedback (bool): True to get compressed feedback with the messages
+            as_client_event (bool): True to get events in client-server format.
         Returns:
             dict: Pagination API results
         """
@@ -98,7 +100,9 @@ class MessageHandler(BaseHandler):
         )
 
         chunk = {
-            "chunk": [self.hs.serialize_event(e) for e in events],
+            "chunk": [
+                self.hs.serialize_event(e, as_client_event) for e in events
+            ],
             "start": pagin_config.from_token.to_string(),
             "end": next_token.to_string(),
         }
@@ -106,7 +110,7 @@ class MessageHandler(BaseHandler):
         defer.returnValue(chunk)
 
     @defer.inlineCallbacks
-    def create_and_send_event(self, event_dict):
+    def create_and_send_event(self, event_dict, ratelimit=True):
         """ Given a dict from a client, create and handle a new event.
 
         Creates an FrozenEvent object, filling out auth_events, prev_events,
@@ -123,7 +127,8 @@ class MessageHandler(BaseHandler):
 
         self.validator.validate_new(builder)
 
-        self.ratelimit(builder.user_id)
+        if ratelimit:
+            self.ratelimit(builder.user_id)
         # TODO(paul): Why does 'event' not have a 'user' object?
         user = self.hs.parse_userid(builder.user_id)
         assert self.hs.is_mine(user), "User must be our own: %s" % (user,)
@@ -152,6 +157,11 @@ class MessageHandler(BaseHandler):
                 context=context,
             )
 
+        if event.type == EventTypes.Message:
+            presence = self.hs.get_handlers().presence_handler
+            with PreserveLoggingContext():
+                presence.bump_presence_active_time(user)
+
         defer.returnValue(event)
 
     @defer.inlineCallbacks
@@ -204,7 +214,7 @@ class MessageHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def snapshot_all_rooms(self, user_id=None, pagin_config=None,
-                           feedback=False):
+                           feedback=False, as_client_event=True):
         """Retrieve a snapshot of all rooms the user is invited or has joined.
 
         This snapshot may include messages for all rooms where the user is
@@ -215,6 +225,7 @@ class MessageHandler(BaseHandler):
             pagin_config (synapse.api.streams.PaginationConfig): The pagination
             config used to determine how many messages *PER ROOM* to return.
             feedback (bool): True to get feedback along with these messages.
+            as_client_event (bool): True to get events in client-server format.
         Returns:
             A list of dicts with "room_id" and "membership" keys for all rooms
             the user is currently invited or joined in on. Rooms where the user
@@ -256,7 +267,7 @@ class MessageHandler(BaseHandler):
             }
 
             if event.membership == Membership.INVITE:
-                d["inviter"] = event.user_id
+                d["inviter"] = event.sender
 
             rooms_ret.append(d)
 
@@ -273,7 +284,10 @@ class MessageHandler(BaseHandler):
                 end_token = now_token.copy_and_replace("room_key", token[1])
 
                 d["messages"] = {
-                    "chunk": [self.hs.serialize_event(m) for m in messages],
+                    "chunk": [
+                        self.hs.serialize_event(m, as_client_event)
+                        for m in messages
+                    ],
                     "start": start_token.to_string(),
                     "end": end_token.to_string(),
                 }

synapse/handlers/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, AuthError, CodeMessageException
-from synapse.api.constants import Membership
+from synapse.api.constants import EventTypes, Membership
 from synapse.util.logcontext import PreserveLoggingContext
 
 from ._base import BaseHandler
@@ -194,6 +194,8 @@ class ProfileHandler(BaseHandler):
         if not self.hs.is_mine(user):
             return
 
+        self.ratelimit(user.to_string())
+
         joins = yield self.store.get_rooms_for_user_where_membership_is(
             user.to_string(),
             [Membership.JOIN],
@@ -201,7 +203,7 @@ class ProfileHandler(BaseHandler):
 
         for j in joins:
             content = {
-                "membership": j.content["membership"],
+                "membership": Membership.JOIN,
             }
 
             yield self.distributor.fire(
@@ -210,9 +212,9 @@ class ProfileHandler(BaseHandler):
 
             msg_handler = self.hs.get_handlers().message_handler
             yield msg_handler.create_and_send_event({
-                "type": j.type,
+                "type": EventTypes.Member,
                 "room_id": j.room_id,
-                "state_key": j.state_key,
+                "state_key": user.to_string(),
                 "content": content,
-                "sender": j.state_key,
-            })
+                "sender": user.to_string()
+            }, ratelimit=False)

synapse/handlers/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -130,6 +130,7 @@ class RoomCreationHandler(BaseHandler):
                 "type": EventTypes.Name,
                 "room_id": room_id,
                 "sender": user_id,
+                "state_key": "",
                 "content": {"name": name},
             })
 
@@ -139,6 +140,7 @@ class RoomCreationHandler(BaseHandler):
                 "type": EventTypes.Topic,
                 "room_id": room_id,
                 "sender": user_id,
+                "state_key": "",
                 "content": {"topic": topic},
             })
 
@@ -243,14 +245,12 @@ class RoomMemberHandler(BaseHandler):
         self.distributor.declare("user_left_room")
 
     @defer.inlineCallbacks
-    def get_room_members(self, room_id, membership=Membership.JOIN):
+    def get_room_members(self, room_id):
         hs = self.hs
 
-        memberships = yield self.store.get_room_members(
-            room_id=room_id, membership=membership
-        )
+        users = yield self.store.get_users_in_room(room_id)
 
-        defer.returnValue([hs.parse_userid(m.user_id) for m in memberships])
+        defer.returnValue([hs.parse_userid(u) for u in users])
 
     @defer.inlineCallbacks
     def fetch_room_distributions_into(self, room_id, localusers=None,
@@ -425,10 +425,22 @@ class RoomMemberHandler(BaseHandler):
             event.room_id,
             self.hs.hostname
         )
+        if not is_host_in_room:
+            # is *anyone* in the room?
+            room_member_keys = [
+                v for (k, v) in context.current_state.keys() if (
+                    k == "m.room.member"
+                )
+            ]
+            if len(room_member_keys) == 0:
+                # has the room been created so we can join it?
+                create_event = context.current_state.get(("m.room.create", ""))
+                if create_event:
+                    is_host_in_room = True
 
         if is_host_in_room:
             should_do_dance = False
-        elif room_host:
+        elif room_host:  # TODO: Shouldn't this be remote_room_host?
             should_do_dance = True
         else:
             # TODO(markjh): get prev_state from snapshot
@@ -442,7 +454,8 @@ class RoomMemberHandler(BaseHandler):
                 should_do_dance = not self.hs.is_mine(inviter)
                 room_host = inviter.domain
             else:
-                should_do_dance = False
+                # return the same error as join_room_alias does
+                raise SynapseError(404, "No known servers")
 
         if should_do_dance:
             handler = self.hs.get_handlers().federation_handler
@@ -529,11 +542,10 @@ class RoomListHandler(BaseHandler):
     def get_public_room_list(self):
         chunk = yield self.store.get_rooms(is_public=True)
         for room in chunk:
-            joined_members = yield self.store.get_room_members(
+            joined_users = yield self.store.get_users_in_room(
                 room_id=room["room_id"],
-                membership=Membership.JOIN
             )
-            room["num_joined_members"] = len(joined_members)
+            room["num_joined_members"] = len(joined_users)
         # FIXME (erikj): START is no longer a valid value
         defer.returnValue({"start": "START", "end": "END", "chunk": chunk})
 

synapse/handlers/typing.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,9 +83,15 @@ class TypingNotificationHandler(BaseHandler):
         if member in self._member_typing_timer:
             self.clock.cancel_call_later(self._member_typing_timer[member])
 
+        def _cb():
+            logger.debug(
+                "%s has timed out in %s", target_user.to_string(), room_id
+            )
+            self._stopped_typing(member)
+
         self._member_typing_until[member] = until
         self._member_typing_timer[member] = self.clock.call_later(
-            timeout / 1000, lambda: self._stopped_typing(member)
+            timeout / 1000.0, _cb
         )
 
         if was_present:
@@ -114,6 +120,10 @@ class TypingNotificationHandler(BaseHandler):
 
         member = RoomMember(room_id=room_id, user=target_user)
 
+        if member in self._member_typing_timer:
+            self.clock.cancel_call_later(self._member_typing_timer[member])
+            del self._member_typing_timer[member]
+
         yield self._stopped_typing(member)
 
     @defer.inlineCallbacks
@@ -136,8 +146,10 @@ class TypingNotificationHandler(BaseHandler):
 
         del self._member_typing_until[member]
 
-        self.clock.cancel_call_later(self._member_typing_timer[member])
-        del self._member_typing_timer[member]
+        if member in self._member_typing_timer:
+            # Don't cancel it - either it already expired, or the real
+            # stopped_typing() will cancel it
+            del self._member_typing_timer[member]
 
     @defer.inlineCallbacks
     def _push_update(self, room_id, user, typing):

synapse/http/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/agent_name.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse import __version__
+
+AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii")

synapse/http/client.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 
+from synapse.http.agent_name import AGENT_NAME
 from twisted.internet import defer, reactor
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError
@@ -51,7 +52,8 @@ class SimpleHttpClient(object):
             "POST",
             uri.encode("ascii"),
             headers=Headers({
-                "Content-Type": ["application/x-www-form-urlencoded"]
+                b"Content-Type": [b"application/x-www-form-urlencoded"],
+                b"User-Agent": [AGENT_NAME],
             }),
             bodyProducer=FileBodyProducer(StringIO(query_bytes))
         )
@@ -86,6 +88,9 @@ class SimpleHttpClient(object):
         response = yield self.agent.request(
             "GET",
             uri.encode("ascii"),
+            headers=Headers({
+                b"User-Agent": [AGENT_NAME],
+            })
         )
 
         body = yield readBody(response)
@@ -108,7 +113,8 @@ class CaptchaServerHttpClient(SimpleHttpClient):
             url.encode("ascii"),
             bodyProducer=FileBodyProducer(StringIO(query_bytes)),
             headers=Headers({
-                "Content-Type": ["application/x-www-form-urlencoded"]
+                b"Content-Type": [b"application/x-www-form-urlencoded"],
+                b"User-Agent": [AGENT_NAME],
             })
         )
 

synapse/http/endpoint.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/matrixfederationclient.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI
 from twisted.web.http_headers import Headers
 from twisted.web._newclient import ResponseDone
 
+from synapse.http.agent_name import AGENT_NAME
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util.async import sleep
 from synapse.util.logcontext import PreserveLoggingContext
@@ -83,7 +84,7 @@ class MatrixFederationHttpClient(object):
                         query_bytes=b"", retry_on_dns_fail=True):
         """ Creates and sends a request to the given url
         """
-        headers_dict[b"User-Agent"] = [b"Synapse"]
+        headers_dict[b"User-Agent"] = [AGENT_NAME]
         headers_dict[b"Host"] = [destination]
 
         url_bytes = urlparse.urlunparse(

synapse/http/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,14 +14,16 @@
 # limitations under the License.
 
 
-from syutil.jsonutil import (
-    encode_canonical_json, encode_pretty_printed_json
-)
+from synapse.http.agent_name import AGENT_NAME
 from synapse.api.errors import (
     cs_exception, SynapseError, CodeMessageException
 )
 from synapse.util.logcontext import LoggingContext
 
+from syutil.jsonutil import (
+    encode_canonical_json, encode_pretty_printed_json
+)
+
 from twisted.internet import defer, reactor
 from twisted.web import server, resource
 from twisted.web.server import NOT_DONE_YET
@@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
 
     request.setResponseCode(code, message=response_code_message)
     request.setHeader(b"Content-Type", b"application/json")
+    request.setHeader(b"Server", AGENT_NAME)
+    request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
 
     if send_cors:
         request.setHeader("Access-Control-Allow-Origin", "*")

synapse/http/server_key_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v0/content_repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/__init__.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import PIL.Image
+
+# check for JPEG support.
+try:
+    PIL.Image._getdecoder("rgb", "jpeg", None)
+except IOError as e:
+    if str(e).startswith("decoder jpeg not available"):
+        raise Exception(
+            "FATAL: jpeg codec not supported. Install pillow correctly! "
+            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+            " pip install pillow --user'"
+        )
+except Exception:
+    # any other exception is fine
+    pass
+
+
+# check for PNG support.
+try:
+    PIL.Image._getdecoder("rgb", "zip", None)
+except IOError as e:
+    if str(e).startswith("decoder zip not available"):
+        raise Exception(
+            "FATAL: zip codec not supported. Install pillow correctly! "
+            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+            " pip install pillow --user'"
+        )
+except Exception:
+    # any other exception is fine
+    pass

synapse/media/v1/base_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -139,6 +139,7 @@ class BaseMediaResource(Resource):
             @download.addBoth
             def callback(media_info):
                 del self.downloads[key]
+                return media_info
         return download
 
     @defer.inlineCallbacks
@@ -201,7 +202,8 @@ class BaseMediaResource(Resource):
         defer.returnValue(media_info)
 
     @defer.inlineCallbacks
-    def _respond_with_file(self, request, media_type, file_path):
+    def _respond_with_file(self, request, media_type, file_path,
+                           file_size=None):
         logger.debug("Responding with %r", file_path)
 
         if os.path.isfile(file_path):
@@ -215,13 +217,20 @@ class BaseMediaResource(Resource):
             request.setHeader(
                 b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
             )
+            if file_size is None:
+                stat = os.stat(file_path)
+                file_size = stat.st_size
+
+            request.setHeader(
+                b"Content-Length", b"%d" % (file_size,)
+            )
 
             with open(file_path, "rb") as f:
                 yield FileSender().beginFileTransfer(f, request)
 
             request.finish()
         else:
-            self._respond_404()
+            self._respond_404(request)
 
     def _get_thumbnail_requirements(self, media_type):
         if media_type == "image/jpeg":

synapse/media/v1/download_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource):
     def _respond_local_file(self, request, media_id):
         media_info = yield self.store.get_local_media(media_id)
         if not media_info:
-            self._respond_404()
+            self._respond_404(request)
             return
 
         media_type = media_info["media_type"]
+        media_length = media_info["media_length"]
         file_path = self.filepaths.local_media_filepath(media_id)
 
-        yield self._respond_with_file(request, media_type, file_path)
+        yield self._respond_with_file(
+            request, media_type, file_path, media_length
+        )
 
     @defer.inlineCallbacks
     def _respond_remote_file(self, request, server_name, media_id):
         media_info = yield self._get_remote_media(server_name, media_id)
 
         media_type = media_info["media_type"]
+        media_length = media_info["media_length"]
         filesystem_id = media_info["filesystem_id"]
 
         file_path = self.filepaths.remote_media_filepath(
             server_name, filesystem_id
         )
 
-        yield self._respond_with_file(request, media_type, file_path)
+        yield self._respond_with_file(
+            request, media_type, file_path, media_length
+        )

synapse/media/v1/filepath.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/media_repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/thumbnail_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource):
             t_type = thumbnail_info["thumbnail_type"]
             t_method = thumbnail_info["thumbnail_method"]
             file_id = thumbnail_info["filesystem_id"]
+            t_length = thumbnail_info["thumbnail_length"]
 
             file_path = self.filepaths.remote_media_thumbnail(
                 server_name, file_id, t_width, t_height, t_type, t_method,
             )
-            yield self._respond_with_file(request, t_type, file_path)
+            yield self._respond_with_file(request, t_type, file_path, t_length)
         else:
             yield self._respond_default_thumbnail(
                 request, media_info, width, height, method, m_type,
@@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource):
         t_height = thumbnail_info["thumbnail_height"]
         t_type = thumbnail_info["thumbnail_type"]
         t_method = thumbnail_info["thumbnail_method"]
+        t_length = thumbnail_info["thumbnail_length"]
 
         file_path = self.filepaths.default_thumbnail(
             top_level_type, sub_type, t_width, t_height, t_type, t_method,
         )
-        yield self.respond_with_file(request, t_type, file_path)
+        yield self.respond_with_file(request, t_type, file_path, t_length)
 
     def _select_thumbnail(self, desired_width, desired_height, desired_method,
                           desired_type, thumbnail_infos):