Updating for the 0.9.6 release.

This commit is contained in:
Brian Lee 2023-09-25 08:03:34 -07:00
parent 66078d82af
commit 47407aefab
6 changed files with 135 additions and 71 deletions

1
.gitignore vendored
View File

@ -0,0 +1 @@
archive

View File

@ -47,24 +47,57 @@ A sample [nginx configuration](docs/examples/nginx_conf.yml) is provided.
For a fully functional production example that includes hosting multiple relays, see this [homelab stack](https://github.com/bleetube/satstack). For a fully functional production example that includes hosting multiple relays, see this [homelab stack](https://github.com/bleetube/satstack).
## Upgrades
Occasionally there are upgrades that require rebuilding the database. You need to `export` before upgrading, and then `import` with the new binary. The role might do the export step, but the import needs to be done manually. Don't rely on the role for the backup. Here's a simple example:
```shell
# Before upgrade
doas -u strfry strfry export > /tmp/backup.jsonl
# After upgrade
systemctl stop strfry
mv strfry-db/data.mdb strfry-db/backup.mdb
cat /tmp/backup.jsonl | doas -u strfry strfry import
doas -u strfry strfry compact strfry-db/compact.mdb
mv strfry-db/compact.mdb strfry-db/data.mdb
systemctl start strfry
```
This is by no means the cleanest way to upgrade, but you get the idea. It's possible to perform the import in a separate process (I think you'd just use a different config file) and then sync the two databases before performing a zero downtime restart.
## Troubleshooting ## Troubleshooting
If `make` fails, try building on a single core:
* If an upgrade fails to build, it could be due to previously built objects. A simple workaround is to delete the strfry source folder `~/src/strfry` and let it try to build from scratch.
* If `make` fails, try building on a single core:
```shell ```shell
ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1' ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1'
``` ```
Logs * Reading your logs:
```shell ```shell
systemctl status strfry systemctl status strfry
journalctl -fu strfry journalctl -fu strfry
``` ```
## Resources ## Maintenance
Plugins: * You should periodically run `compact` on your strfry database.
```shell
systemctl stop strfry
doas -u strfry strfry compact strfry-db/compact.mdb
mv strfry-db/compact.mdb strfry-db/data.mdb
systemctl start strfry
```
* You can prune events from the database, reducing it's size will reduce the overall compute load on the relay. Make a backup beforehand. Here is a simple example of deleting events that are more than 90 days old:
```shell
doas -u strfry strfry export > /tmp/backup.jsonl
doas -u strfry strfry delete --age=$((90 * 24 * 60 * 60))
```
For a more advanced pruning strategy, you can implement an export/import process to remove certain kinds of events more aggresively. See [bleetube/strfry-prune](https://github.com/bleetube/strfry-prune) for an example.
* [strfry policies](https://gitlab.com/soapbox-pub/strfry-policies) plugin (deno) - recommended
* [strfry metrics](https://github.com/bleetube/strfry-plugin) (python)
* [spamblaster](https://github.com/relaytools/spamblaster) (go)

View File

@ -18,6 +18,31 @@ strfry_dbParams:
# Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
mapsize: 10995116277760 mapsize: 10995116277760
# Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
noReadAhead: no
strfry_events:
# Maximum size of normalised JSON, in bytes
maxEventSize: 65536
# Events newer than this will be rejected
rejectEventsNewerThanSeconds: 900
# Events older than this will be rejected
rejectEventsOlderThanSeconds: 94608000
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds: 60
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds: 300
# Maximum number of tags allowed
maxNumTags: 2000
# Maximum size for tag values, in bytes
maxTagValSize: 1024
strfry_relay: strfry_relay:
# Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required)
bind: "127.0.0.1" bind: "127.0.0.1"
@ -66,9 +91,6 @@ strfry_relay:
# If non-empty, path to an executable script that implements the writePolicy plugin logic # If non-empty, path to an executable script that implements the writePolicy plugin logic
plugin: "{{ strfry_data_path }}/strfry-policy.ts" plugin: "{{ strfry_data_path }}/strfry-policy.ts"
# Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
lookbackSeconds: 0
compression: compression:
# Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required) # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required)
enabled: yes enabled: yes
@ -89,6 +111,9 @@ strfry_relay:
# Log performance metrics for initial REQ database scans # Log performance metrics for initial REQ database scans
dbScanPerf: no dbScanPerf: no
# Log reason for invalid event rejection? Can be disabled to silence excessive logging
invalidEvents: yes
numThreads: numThreads:
# Ingester threads: route incoming requests, validate events/sigs (restart required) # Ingester threads: route incoming requests, validate events/sigs (restart required)
ingester: 3 ingester: 3
@ -99,27 +124,12 @@ strfry_relay:
# reqMonitor threads: Handle filtering of new events (restart required) # reqMonitor threads: Handle filtering of new events (restart required)
reqMonitor: 3 reqMonitor: 3
# yesstr threads: Experimental yesstr protocol (restart required) # negentropy threads: Handle negentropy protocol messages (restart required)
yesstr: 1 negentropy: 2
strfry_events: negentropy:
# Maximum size of normalised JSON, in bytes # Support negentropy protocol messages
maxEventSize: 65536 enabled: yes
# Events newer than this will be rejected # Maximum records that sync will process before returning an error
rejectEventsNewerThanSeconds: 900 maxSyncEvents: 1000000
# Events older than this will be rejected
rejectEventsOlderThanSeconds: 94608000
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds: 60
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds: 300
# Maximum number of tags allowed
maxNumTags: 2000
# Maximum size for tag values, in bytes
maxTagValSize: 1024

View File

@ -16,19 +16,28 @@
ansible.builtin.command: ansible.builtin.command:
cmd: git submodule update --init cmd: git submodule update --init
chdir: "{{ ansible_env.HOME }}/src/strfry" chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed when: git_repository.changed
changed_when: git_repository.changed changed_when: git_repository.changed
- name: Run make setup-golpe - name: Run make setup-golpe
ansible.builtin.command: ansible.builtin.command:
cmd: make setup-golpe cmd: make setup-golpe
chdir: "{{ ansible_env.HOME }}/src/strfry" chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed when: git_repository.changed
changed_when: git_repository.changed changed_when: git_repository.changed
- name: Build strfry - name: Build strfry
ansible.builtin.command: ansible.builtin.command:
cmd: "make -j{{ strfry_make_jobs|default(1) }}" cmd: "make -j{{ strfry_make_jobs|default(1) }}"
chdir: "{{ ansible_env.HOME }}/src/strfry" chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed when: git_repository.changed
changed_when: git_repository.changed
- name: Backup existing strfry-db
ansible.builtin.shell:
cmd: strfry export > backup.jsonl
chdir: "{{ strfry_binary_path }}"
become: yes
become_user: "{{ strfry_system_user }}"
when: git_repository.changed
changed_when: git_repository.changed changed_when: git_repository.changed

View File

@ -5,6 +5,7 @@
dest: "{{ strfry_binary_path }}" dest: "{{ strfry_binary_path }}"
mode: 0755 mode: 0755
remote_src: true remote_src: true
notify: restart strfry
- name: Setup strfry service unit - name: Setup strfry service unit
ansible.builtin.template: ansible.builtin.template:
@ -34,7 +35,7 @@
dest: "{{ strfry_relay.writePolicy.plugin }}" dest: "{{ strfry_relay.writePolicy.plugin }}"
owner: "{{ strfry_system_user }}" owner: "{{ strfry_system_user }}"
group: "{{ strfry_system_group }}" group: "{{ strfry_system_group }}"
# force: false # Never overwrite, this is just a starter policy force: false # Never overwrite, this is just a starter policy
mode: '0755' mode: '0755'
when: strfry_policies_enabled when: strfry_policies_enabled
notify: restart strfry notify: restart strfry

View File

@ -13,6 +13,31 @@ dbParams {
# Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
mapsize = {{ strfry_dbParams.mapsize }} mapsize = {{ strfry_dbParams.mapsize }}
# Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
noReadAhead = {{ "true" if strfry_dbParams.noReadAhead else "false" }}
}
events {
# Maximum size of normalised JSON, in bytes
maxEventSize = {{ strfry_events.maxEventSize }}
# Events newer than this will be rejected
rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }}
# Events older than this will be rejected
rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
# Maximum number of tags allowed
maxNumTags = {{ strfry_events.maxNumTags }}
# Maximum size for tag values, in bytes
maxTagValSize = {{ strfry_events.maxTagValSize }}
} }
relay { relay {
@ -63,9 +88,6 @@ relay {
writePolicy { writePolicy {
# If non-empty, path to an executable script that implements the writePolicy plugin logic # If non-empty, path to an executable script that implements the writePolicy plugin logic
plugin = "{{ strfry_relay.writePolicy.plugin }}" plugin = "{{ strfry_relay.writePolicy.plugin }}"
# Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
lookbackSeconds = {{ strfry_relay.writePolicy.lookbackSeconds }}
} }
compression { compression {
@ -88,6 +110,9 @@ relay {
# Log performance metrics for initial REQ database scans # Log performance metrics for initial REQ database scans
dbScanPerf = {{ "true" if strfry_relay.logging.dbScanPerf else "false" }} dbScanPerf = {{ "true" if strfry_relay.logging.dbScanPerf else "false" }}
# Log reason for invalid event rejection? Can be disabled to silence excessive logging
invalidEvents = {{ "true" if strfry_relay.logging.invalidEvents else "false" }}
} }
numThreads { numThreads {
@ -100,30 +125,15 @@ relay {
# reqMonitor threads: Handle filtering of new events (restart required) # reqMonitor threads: Handle filtering of new events (restart required)
reqMonitor = {{ strfry_relay.numThreads.reqMonitor }} reqMonitor = {{ strfry_relay.numThreads.reqMonitor }}
# yesstr threads: Experimental yesstr protocol (restart required) # negentropy threads: Handle negentropy protocol messages (restart required)
yesstr = {{ strfry_relay.numThreads.yesstr }} negentropy = {{ strfry_relay.numThreads.negentropy }}
}
} }
events { negentropy {
# Maximum size of normalised JSON, in bytes # Support negentropy protocol messages
maxEventSize = {{ strfry_events.maxEventSize }} enabled = {{ "true" if strfry_relay.negentropy.enabled else "false" }}
# Events newer than this will be rejected # Maximum records that sync will process before returning an error
rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }} maxSyncEvents = {{ strfry_relay.negentropy.maxSyncEvents }}
}
# Events older than this will be rejected
rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
# Maximum number of tags allowed
maxNumTags = {{ strfry_events.maxNumTags }}
# Maximum size for tag values, in bytes
maxTagValSize = {{ strfry_events.maxTagValSize }}
} }