Jason/debug phone home stats weirds

.ci/scripts/calculate_jobs.py

@@ -63,7 +63,7 @@ def set_output(key: str, value: str):
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.11", "3.12", "3.13")
+        for version in ("3.11", "3.12")
     )
 
 # Only test postgres against the earliest and latest Python versions that we
@@ -76,7 +76,7 @@ def set_output(key: str, value: str):
         "extras": "all",
     },
     {
-        "python-version": "3.14",
+        "python-version": "3.13",
         "database": "postgres",
         "postgres-version": "17",
         "extras": "all",

.github/workflows/famedly-tests.yml

@@ -216,7 +216,91 @@ jobs:
             --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.job.postgres-version }}
+            postgres:${{ matrix.job.postgres-version }} -c full_page_writes=off -c wal_init_zero=off -c effective_io_concurrency=200 -c maintenance_io_concurrency=200 -c max_worker_processes=4 -c max_parallel_workers=4 -c random_page_cost=1.05
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - uses: matrix-org/setup-python-poetry@v2
+        with:
+          python-version: ${{ matrix.job.python-version }}
+          poetry-version: "2.1.1"
+          extras: ${{ matrix.job.extras }}
+      - name: Await PostgreSQL
+        if: ${{ matrix.job.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      # coverage is already installed from the pyproject.toml file
+      - run: poetry run pip install coverage-enable-subprocess
+        # Normally, this Github runner has 4 cores(vCPUs) available to it.
+        # Using only one trial test runner does not saturate the cpu. Rough experimentation in the past led to having 3
+        # of these runners per core as optimal, but now coverage is required and there may be the need
+        # to scale back a little. This should be 8 trial test runners/jobs. Adding in
+        # --debug=core will display which measurement core is in use. For python 3.14+
+        # this is educational.
+      - run: poetry run coverage run --debug=core -m twisted.trial -j$((`nproc` * 2 )) tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+      - run: poetry run coverage combine
+      - run: poetry run coverage xml
+      - name: Codecov - Upload coverage
+        uses: codecov/codecov-action@v5
+        with:
+          use_oidc: true
+
+  trial-future:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs:
+      - calculate-test-jobs
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    strategy:
+      matrix:
+        job:
+          - python-version: 3.14
+            database: SQLite
+            extras: all
+
+          - python-version: 3.14
+            database: postgres
+            postgres-version: 18
+            extras: all
+    env:
+      # Both of these are used for the coverage system. TOP gives the directory that is
+      # the root of the source code, and COVERAGE_PROCESS_START points at the coverage
+      # settings file to use(which also enables multiprocess implicit mode for coverage.py)
+      TOP: ${{ github.workspace }}
+      COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
+        if: ${{ matrix.job.postgres-version }}
+        # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
+        # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
+        run: |
+          docker run -d -p 5432:5432 \
+            --tmpfs /var/lib/postgresql/:rw,size=6144m \
+            --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.job.postgres-version }} -c full_page_writes=off -c wal_init_zero=off -c effective_io_concurrency=200 -c maintenance_io_concurrency=200 -c max_worker_processes=4 -c max_parallel_workers=4 -c random_page_cost=1.05
 
       - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
 

CHANGES.md

@@ -10,8 +10,8 @@ No significant changes since 1.143.0rc2.
 
 
 ### Famedly additions for v1.143.0_1
-- fix: docs url broken on /version ([\#219](https://github.com/famedly/synapse/pull/219)) (FrenchgGithubUser)
-- fix: provide remote servers a way to find out about an event created during the remote join handshake ([\#71](https://github.com/famedly/synapse/pull/71)) (FrenchgGithubUser)
+- fix: docs url broken on /version ([\#219](https://github.com/famedly/synapse/pull/219)) (FrenchGithubUser)
+- fix: provide remote servers a way to find out about an event created during the remote join handshake ([\#71](https://github.com/famedly/synapse/pull/71)) (FrenchGithubUser)
 
 
 # synapse 1.143.0rc2 (2025-11-18)

synapse/app/phone_stats_home.py

@@ -118,13 +118,36 @@ async def _phone_stats_home(
         stats["memory_rss"] = new[1].ru_maxrss
 
         # Get CPU time in % of a single core, not % of all cores
+        logger.info("Calculating used_cpu_time")
+        logger.info(
+            "new_cpu_time: %r + %r = %r",
+            new[1].ru_utime,
+            new[1].ru_stime,
+            new[1].ru_utime + new[1].ru_stime,
+        )
+        logger.info(
+            "old_cpu_time: %r + %r = %r",
+            old[1].ru_utime,
+            old[1].ru_stime,
+            old[1].ru_utime + old[1].ru_stime,
+        )
+        logger.info(
+            "difference: %r",
+            (new[1].ru_utime + new[1].ru_stime) - (old[1].ru_utime + old[1].ru_stime),
+        )
         used_cpu_time = (new[1].ru_utime + new[1].ru_stime) - (
             old[1].ru_utime + old[1].ru_stime
         )
+        logger.info("new(time): %r - previous(time): %r =:", new[0], old[0])
+        time_diff = new[0] - old[0]
+        logger.info("time_diff: %r ", time_diff)
         if used_cpu_time == 0 or new[0] == old[0]:
+            logger.info("calculated cpu time: 0, ignoring value as times are same")
             stats["cpu_average"] = 0
         else:
-            stats["cpu_average"] = math.floor(used_cpu_time / (new[0] - old[0]) * 100)
+            the_number = math.floor(used_cpu_time / time_diff) * 100
+            logger.info("calculated cpu time: %r", the_number)
+            stats["cpu_average"] = the_number
 
         #
         # General statistics

tests/test_phone_home.py

@@ -31,7 +31,7 @@
 from synapse.types import JsonDict
 from synapse.util.clock import Clock
 
-from tests.unittest import HomeserverTestCase
+from tests.unittest import INFO, HomeserverTestCase
 
 
 class PhoneHomeStatsTestCase(HomeserverTestCase):
@@ -46,6 +46,7 @@ def test_performance_frozen_clock(self) -> None:
         self.get_success(phone_stats_home(self.hs, stats, past_stats))
         self.assertEqual(stats["cpu_average"], 0)
 
+    @INFO
     def test_performance_100(self) -> None:
         """
         1 second of usage over 1 second is 100% CPU usage.