STASH

scripts-dev/workers_setup.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # Copyright 2022 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -59,7 +60,7 @@ def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]:
     result = []
     worker_overall_num = 0
     for worker_type, worker_type_count in workers:
-        for worker_idx in range(worker_type_count):
+        for worker_idx in range(1, worker_type_count + 1):
             worker_overall_num += 1
             if worker_type == "main":
                 worker_name = "main"
@@ -73,7 +74,7 @@ def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]:
     return result
 
 
-def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> None:
+def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path, server_name: str) -> None:
     if target_path.exists():
         print("Target path already exists. Won't overwrite.")
         return
@@ -85,6 +86,8 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N
     with open(target_path.joinpath("signing.key"), "w") as fout:
         write_signing_keys(fout, key)
 
+    macaroon_secret_key = random_string(32)
+
     env = Environment(loader=FileSystemLoader(dirname(__file__) + "/workers_setup"))
     hs_template = env.get_template("homeserver.yaml.j2")
     worker_template = env.get_template("worker.yaml.j2")
@@ -92,6 +95,8 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N
 
     worker_dir = target_path.joinpath("workers")
     worker_dir.mkdir()
+    worker_logging_dir = target_path.joinpath("workers.logging")
+    worker_logging_dir.mkdir()
     worker_dir = worker_dir.resolve()
 
     logs_dir = target_path.joinpath("logs")
@@ -102,6 +107,21 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N
     workers_by_name = {worker.name: worker for worker in all_workers}
 
     for worker in all_workers:
+        log_config_path = worker_logging_dir.joinpath(f"{worker.name}.logging.yaml")
+        log_config = logging_template.render(
+            worker=worker,
+            worker_dir=worker_dir,
+            logs_dir=logs_dir,
+            all_workers=all_workers,
+            workers_by_name=workers_by_name
+        )
+        with open(log_config_path, "w") as fout:
+            fout.write(log_config)
+
+        if worker.name == "main":
+            # Main can't use a worker file.
+            continue
+
         worker_config_path = worker_dir.joinpath(f"{worker.name}.yaml")
         worker_config = worker_template.render(
             worker=worker,
@@ -113,31 +133,23 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N
         with open(worker_config_path, "w") as fout:
             fout.write(worker_config)
 
-        log_config_path = worker_dir.joinpath(f"{worker.name}.logging.yaml")
-        log_config = logging_template.render(
-            worker=worker,
-            worker_dir=worker_dir,
-            logs_dir=logs_dir,
-            all_workers=all_workers,
-            workers_by_name=workers_by_name
-        )
-        with open(log_config_path, "w") as fout:
-            fout.write(log_config)
-
     hs_config_path = target_path.joinpath("homeserver.yaml")
     hs_config = hs_template.render(
         all_workers=all_workers,
         worker_dir=worker_dir,
-        logs_dir=logs_dir
+        logs_dir=logs_dir,
+        server_name=server_name,
+        macaroon_secret_key=macaroon_secret_key
     )
     with open(hs_config_path, "w") as fout:
         fout.write(hs_config)
 
 
-def main(target_path: Path) -> None:
-    generate(DESIRED_WORKERS, target_path)
+def main(target_path: Path, server_name: str) -> None:
+    generate(DESIRED_WORKERS, target_path, server_name)
 
 
 if __name__ == '__main__':
     target_path = Path(sys.argv[1])
-    main(target_path)
+    server_name = sys.argv[2]
+    main(target_path, server_name)

scripts-dev/workers_setup/homeserver.yaml.j2

@@ -1,38 +1,72 @@
 
+server_name: {{ server_name }}
+
+report_stats: false
+
+signing_key_path: "{{ worker_dir }}/../signing.key"
+
+macaroon_secret_key: "{{ macaroon_secret_key }}"
+
+redis:
+  enabled: true
+  #host: localhost
+  #port: 6379
+
+trusted_key_servers: []
+
+database:
+  name: psycopg2
+  args:
+    # Comment out user, password and host to use UNIX socket auth.
+    # For testing, create a database owned by your Postgres user that is logged
+    # in with your UNIX user
+    #user: "synapse"
+    #password:
+    database: "{{ server_name }}"
+    #host: "localhost"
+    cp_min: 5
+    cp_max: 10
+
 instance_map:
-{% for worker in all_workers %}
+{-% for worker in all_workers %}
   {{ worker.name }}:
     host: {{ worker.ip }}
     port: 9090
 {% endfor %}
 
-
stream_writers:
+stream_writers:
   events:
-{% for worker in all_workers %}
-{% if worker.kind == "event_persister" %}
+{% for worker in all_workers %-}
+{-% if worker.kind == "event_persister" %}
     - {{ worker.name }}
-{% endif %}
+{% endif %-}
+{-% endfor %}
 
-  typing:
-{% for worker in all_workers %}
-{% if worker.kind == "typing" %}
+typing:
+{% for worker in all_workers %-}
+{-% if worker.kind == "typing" %}
     - {{ worker.name }}
-{% endif %}
+{% endif %-}
+{-% endfor %}
 
 
 start_pushers: false
 
 pusher_instances:
-{% if worker.kind == "pusher" %}
+{% for worker in all_workers %-}
+{-% if worker.kind == "pusher" %}
     - {{ worker.name }}
-{% endif %}
+{% endif %-}
+{-% endfor %}
 
 notify_appservices: False
 
 federation_sender_instances:
-{% if worker.kind == "federation_sender" %}
+{% for worker in all_workers %-}
+{-% if worker.kind == "federation_sender" %}
     - {{ worker.name }}
-{% endif %}
+{% endif %-}
+{-% endfor %}
 
 enable_media_repo: False
 
@@ -40,3 +74,4 @@ media_instance_running_background_jobs: "media1"
 
 update_user_directory: False
 
+pid_file: "{{ logs_dir }}/main.pid"

scripts-dev/workers_setup/worker.yaml.j2

@@ -3,23 +3,23 @@
 worker_app: synapse.app.homeserver
 {% else %}
 worker_app: synapse.app.generic_worker
-worker_name: {{ worker_name }}
-
# The replication listener on the main synapse process.
-worker_replication_host: {{ worker_ip }}
+worker_name: {{ worker.name }}
+# The replication listener on the main synapse process.
+worker_replication_host: {{ worker.ip }}
 worker_replication_http_port: 9090
-
worker_listeners:
+worker_listeners:
   - type: http
     port: 8080
     resources:
     - names:
-    - client
-    - federation
+      - client
+      - federation
 {% if worker.kind == "media" %}
-    - media
+      - media
 {% endif %}
 
-worker_log_config: '{{ workers_dir }}/{{ worker_name }}.logging.yaml'
-worker_pid_file: '{{ logs_dir }}/{{ worker_name }}.pid'
+worker_log_config: '{{ worker_dir }}.logging/{{ worker.name }}.logging.yaml'
+worker_pid_file: '{{ logs_dir }}/{{ worker.name }}.pid'
 {% endif %}
 
 {% set main_worker = workers_by_name.main %}