Updating for the 0.9.6 release.

.gitignore

@@ -0,0 +1 @@
+archive

README.md

@@ -47,24 +47,57 @@ A sample [nginx configuration](docs/examples/nginx_conf.yml) is provided.
 
 For a fully functional production example that includes hosting multiple relays, see this [homelab stack](https://github.com/bleetube/satstack).
 
+## Upgrades
+
+Occasionally there are upgrades that require rebuilding the database. You need to `export` before upgrading, and then `import` with the new binary. The role might do the export step, but the import needs to be done manually. Don't rely on the role for the backup. Here's a simple example:
+
+```shell
+# Before upgrade
+doas -u strfry strfry export > /tmp/backup.jsonl
+# After upgrade
+systemctl stop strfry
+mv strfry-db/data.mdb strfry-db/backup.mdb
+cat /tmp/backup.jsonl | doas -u strfry strfry import
+doas -u strfry strfry compact strfry-db/compact.mdb
+mv strfry-db/compact.mdb strfry-db/data.mdb
+systemctl start strfry
+```
+
+This is by no means the cleanest way to upgrade, but you get the idea. It's possible to perform the import in a separate process (I think you'd just use a different config file) and then sync the two databases before performing a zero downtime restart.
+
 ## Troubleshooting
-If `make` fails, try building on a single core:
 
-```shell
+* If an upgrade fails to build, it could be due to previously built objects. A simple workaround is to delete the strfry source folder `~/src/strfry` and let it try to build from scratch.
-ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1'
-```
 
-Logs
+* If `make` fails, try building on a single core:
 
-```shell
+  ```shell
-systemctl status strfry
+  ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1'
-journalctl -fu strfry
+  ```
-```
 
-## Resources
+* Reading your logs:
 
-Plugins:
+  ```shell
+  systemctl status strfry
+  journalctl -fu strfry
+  ```
 
-* [strfry policies](https://gitlab.com/soapbox-pub/strfry-policies) plugin (deno) - recommended
+## Maintenance
-* [strfry metrics](https://github.com/bleetube/strfry-plugin) (python)
+
-* [spamblaster](https://github.com/relaytools/spamblaster) (go)
+* You should periodically run `compact` on your strfry database.
+
+  ```shell
+  systemctl stop strfry
+  doas -u strfry strfry compact strfry-db/compact.mdb
+  mv strfry-db/compact.mdb strfry-db/data.mdb
+  systemctl start strfry
+  ```
+
+* You can prune events from the database, reducing it's size will reduce the overall compute load on the relay. Make a backup beforehand. Here is a simple example of deleting events that are more than 90 days old:
+
+  ```shell
+  doas -u strfry strfry export > /tmp/backup.jsonl
+  doas -u strfry strfry delete --age=$((90 * 24 * 60 * 60))
+  ```
+  For a more advanced pruning strategy, you can implement an export/import process to remove certain kinds of events more aggresively. See [bleetube/strfry-prune](https://github.com/bleetube/strfry-prune) for an example.
+  

defaults/main.yml

@@ -18,6 +18,31 @@ strfry_dbParams:
   # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
   mapsize: 10995116277760
 
+  # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
+  noReadAhead: no
+
+strfry_events:
+  # Maximum size of normalised JSON, in bytes
+  maxEventSize: 65536
+
+  # Events newer than this will be rejected
+  rejectEventsNewerThanSeconds: 900
+
+  # Events older than this will be rejected
+  rejectEventsOlderThanSeconds: 94608000
+
+  # Ephemeral events older than this will be rejected
+  rejectEphemeralEventsOlderThanSeconds: 60
+
+  # Ephemeral events will be deleted from the DB when older than this
+  ephemeralEventsLifetimeSeconds: 300
+
+  # Maximum number of tags allowed
+  maxNumTags: 2000
+
+  # Maximum size for tag values, in bytes
+  maxTagValSize: 1024
+
 strfry_relay:
   # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required)
   bind: "127.0.0.1"
@@ -66,9 +91,6 @@ strfry_relay:
     # If non-empty, path to an executable script that implements the writePolicy plugin logic
     plugin: "{{ strfry_data_path }}/strfry-policy.ts"
 
-    # Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
-    lookbackSeconds: 0
-
   compression:
     # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required)
     enabled: yes
@@ -89,6 +111,9 @@ strfry_relay:
     # Log performance metrics for initial REQ database scans
     dbScanPerf: no
 
+    # Log reason for invalid event rejection? Can be disabled to silence excessive logging
+    invalidEvents: yes
+
   numThreads:
     # Ingester threads: route incoming requests, validate events/sigs (restart required)
     ingester: 3
@@ -99,27 +124,12 @@ strfry_relay:
     # reqMonitor threads: Handle filtering of new events (restart required)
     reqMonitor: 3
 
-    # yesstr threads: Experimental yesstr protocol (restart required)
+    # negentropy threads: Handle negentropy protocol messages (restart required)
-    yesstr: 1
+    negentropy: 2
 
-strfry_events:
+  negentropy:
-  # Maximum size of normalised JSON, in bytes
+    # Support negentropy protocol messages
-  maxEventSize: 65536
+    enabled: yes
 
-  # Events newer than this will be rejected
+    # Maximum records that sync will process before returning an error
-  rejectEventsNewerThanSeconds: 900
+    maxSyncEvents: 1000000
-
-  # Events older than this will be rejected
-  rejectEventsOlderThanSeconds: 94608000
-
-  # Ephemeral events older than this will be rejected
-  rejectEphemeralEventsOlderThanSeconds: 60
-
-  # Ephemeral events will be deleted from the DB when older than this
-  ephemeralEventsLifetimeSeconds: 300
-
-  # Maximum number of tags allowed
-  maxNumTags: 2000
-
-  # Maximum size for tag values, in bytes
-  maxTagValSize: 1024

tasks/build.yml

@@ -16,19 +16,28 @@
   ansible.builtin.command:
     cmd: git submodule update --init
     chdir: "{{ ansible_env.HOME }}/src/strfry"
-# when: git_repository.changed
+  when: git_repository.changed
   changed_when: git_repository.changed
 
 - name: Run make setup-golpe
   ansible.builtin.command:
     cmd: make setup-golpe
     chdir: "{{ ansible_env.HOME }}/src/strfry"
-# when: git_repository.changed
+  when: git_repository.changed
   changed_when: git_repository.changed
 
 - name: Build strfry
   ansible.builtin.command:
     cmd: "make -j{{ strfry_make_jobs|default(1) }}"
     chdir: "{{ ansible_env.HOME }}/src/strfry"
-# when: git_repository.changed
+  when: git_repository.changed
+  changed_when: git_repository.changed
+
+- name: Backup existing strfry-db
+  ansible.builtin.shell:
+    cmd: strfry export > backup.jsonl
+    chdir: "{{ strfry_binary_path }}"
+  become: yes
+  become_user: "{{ strfry_system_user }}"
+  when: git_repository.changed
   changed_when: git_repository.changed

tasks/install.yml

@@ -5,6 +5,7 @@
     dest: "{{ strfry_binary_path }}"
     mode: 0755
     remote_src: true
+  notify: restart strfry
 
 - name: Setup strfry service unit
   ansible.builtin.template:
@@ -34,7 +35,7 @@
     dest: "{{ strfry_relay.writePolicy.plugin }}"
     owner: "{{ strfry_system_user }}"
     group: "{{ strfry_system_group }}"
-#   force: false # Never overwrite, this is just a starter policy
+    force: false # Never overwrite, this is just a starter policy
     mode: '0755'
   when: strfry_policies_enabled
   notify: restart strfry

templates/strfry.conf

@@ -13,6 +13,31 @@ dbParams {
 
     # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
     mapsize = {{ strfry_dbParams.mapsize }}
+
+    # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
+    noReadAhead = {{ "true" if strfry_dbParams.noReadAhead else "false" }}
+}
+events {
+    # Maximum size of normalised JSON, in bytes
+    maxEventSize = {{ strfry_events.maxEventSize }}
+
+    # Events newer than this will be rejected
+    rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }}
+
+    # Events older than this will be rejected
+    rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
+
+    # Ephemeral events older than this will be rejected
+    rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
+
+    # Ephemeral events will be deleted from the DB when older than this
+    ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
+
+    # Maximum number of tags allowed
+    maxNumTags = {{ strfry_events.maxNumTags }}
+
+    # Maximum size for tag values, in bytes
+    maxTagValSize = {{ strfry_events.maxTagValSize }}
 }
 
 relay {
@@ -63,9 +88,6 @@ relay {
     writePolicy {
         # If non-empty, path to an executable script that implements the writePolicy plugin logic
         plugin = "{{ strfry_relay.writePolicy.plugin }}"
-
-        # Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
-        lookbackSeconds = {{ strfry_relay.writePolicy.lookbackSeconds }}
     }
 
     compression {
@@ -88,6 +110,9 @@ relay {
 
         # Log performance metrics for initial REQ database scans
         dbScanPerf = {{ "true" if strfry_relay.logging.dbScanPerf else "false" }}
+
+        # Log reason for invalid event rejection? Can be disabled to silence excessive logging
+        invalidEvents = {{ "true" if strfry_relay.logging.invalidEvents else "false" }}
     }
 
     numThreads {
@@ -100,30 +125,15 @@ relay {
         # reqMonitor threads: Handle filtering of new events (restart required)
         reqMonitor = {{ strfry_relay.numThreads.reqMonitor }}
 
-        # yesstr threads: Experimental yesstr protocol (restart required)
+        # negentropy threads: Handle negentropy protocol messages (restart required)
-        yesstr = {{ strfry_relay.numThreads.yesstr }}
+        negentropy = {{ strfry_relay.numThreads.negentropy }}
+    }
+
+    negentropy {
+        # Support negentropy protocol messages
+        enabled = {{ "true" if strfry_relay.negentropy.enabled else "false" }}
+
+        # Maximum records that sync will process before returning an error
+        maxSyncEvents = {{ strfry_relay.negentropy.maxSyncEvents }}
     }
 }
-
-events {
-    # Maximum size of normalised JSON, in bytes
-    maxEventSize = {{ strfry_events.maxEventSize }}
-
-    # Events newer than this will be rejected
-    rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }}
-
-    # Events older than this will be rejected
-    rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
-
-    # Ephemeral events older than this will be rejected
-    rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
-
-    # Ephemeral events will be deleted from the DB when older than this
-    ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
-
-    # Maximum number of tags allowed
-    maxNumTags = {{ strfry_events.maxNumTags }}
-
-    # Maximum size for tag values, in bytes
-    maxTagValSize = {{ strfry_events.maxTagValSize }}
-}