pleb/ansible-role-strfry
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updating for the 0.9.6 release.

Browse Source
This commit is contained in:
Brian Lee 2023-09-25 08:03:34 -07:00
parent 66078d82af
commit 47407aefab
6 changed files with 135 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -0,0 +1 @@
archive

61
README.md
View File

@ -47,24 +47,57 @@ A sample [nginx configuration](docs/examples/nginx_conf.yml) is provided.
For a fully functional production example that includes hosting multiple relays, see this [homelab stack](https://github.com/bleetube/satstack).
## Upgrades
Occasionally there are upgrades that require rebuilding the database. You need to `export` before upgrading, and then `import` with the new binary. The role might do the export step, but the import needs to be done manually. Don't rely on the role for the backup. Here's a simple example:
```shell
# Before upgrade
doas -u strfry strfry export > /tmp/backup.jsonl
# After upgrade
systemctl stop strfry
mv strfry-db/data.mdb strfry-db/backup.mdb
cat /tmp/backup.jsonl | doas -u strfry strfry import
doas -u strfry strfry compact strfry-db/compact.mdb
mv strfry-db/compact.mdb strfry-db/data.mdb
systemctl start strfry
```
This is by no means the cleanest way to upgrade, but you get the idea. It's possible to perform the import in a separate process (I think you'd just use a different config file) and then sync the two databases before performing a zero downtime restart.
## Troubleshooting
If `make` fails, try building on a single core:
```shell
ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1'
```
* If an upgrade fails to build, it could be due to previously built objects. A simple workaround is to delete the strfry source folder `~/src/strfry` and let it try to build from scratch.
Logs
* If `make` fails, try building on a single core:
```shell
systemctl status strfry
journalctl -fu strfry
```
```shell
ansible-playbook playbooks/strfry/main.yml -e 'strfry_make_jobs=1'
```
## Resources
* Reading your logs:
Plugins:
```shell
systemctl status strfry
journalctl -fu strfry
```
* [strfry policies](https://gitlab.com/soapbox-pub/strfry-policies) plugin (deno) - recommended
* [strfry metrics](https://github.com/bleetube/strfry-plugin) (python)
* [spamblaster](https://github.com/relaytools/spamblaster) (go)
## Maintenance
* You should periodically run `compact` on your strfry database.
```shell
systemctl stop strfry
doas -u strfry strfry compact strfry-db/compact.mdb
mv strfry-db/compact.mdb strfry-db/data.mdb
systemctl start strfry
```
* You can prune events from the database, reducing it's size will reduce the overall compute load on the relay. Make a backup beforehand. Here is a simple example of deleting events that are more than 90 days old:
```shell
doas -u strfry strfry export > /tmp/backup.jsonl
doas -u strfry strfry delete --age=$((90 * 24 * 60 * 60))
```
For a more advanced pruning strategy, you can implement an export/import process to remove certain kinds of events more aggresively. See [bleetube/strfry-prune](https://github.com/bleetube/strfry-prune) for an example.

60
defaults/main.yml
View File

@ -18,6 +18,31 @@ strfry_dbParams:
# Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
mapsize: 10995116277760
# Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
noReadAhead: no
strfry_events:
# Maximum size of normalised JSON, in bytes
maxEventSize: 65536
# Events newer than this will be rejected
rejectEventsNewerThanSeconds: 900
# Events older than this will be rejected
rejectEventsOlderThanSeconds: 94608000
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds: 60
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds: 300
# Maximum number of tags allowed
maxNumTags: 2000
# Maximum size for tag values, in bytes
maxTagValSize: 1024
strfry_relay:
# Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required)
bind: "127.0.0.1"
@ -66,9 +91,6 @@ strfry_relay:
# If non-empty, path to an executable script that implements the writePolicy plugin logic
plugin: "{{ strfry_data_path }}/strfry-policy.ts"
# Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
lookbackSeconds: 0
compression:
# Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required)
enabled: yes
@ -89,6 +111,9 @@ strfry_relay:
# Log performance metrics for initial REQ database scans
dbScanPerf: no
# Log reason for invalid event rejection? Can be disabled to silence excessive logging
invalidEvents: yes
numThreads:
# Ingester threads: route incoming requests, validate events/sigs (restart required)
ingester: 3
@ -99,27 +124,12 @@ strfry_relay:
# reqMonitor threads: Handle filtering of new events (restart required)
reqMonitor: 3
# yesstr threads: Experimental yesstr protocol (restart required)
yesstr: 1
# negentropy threads: Handle negentropy protocol messages (restart required)
negentropy: 2
strfry_events:
# Maximum size of normalised JSON, in bytes
maxEventSize: 65536
negentropy:
# Support negentropy protocol messages
enabled: yes
# Events newer than this will be rejected
rejectEventsNewerThanSeconds: 900
# Events older than this will be rejected
rejectEventsOlderThanSeconds: 94608000
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds: 60
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds: 300
# Maximum number of tags allowed
maxNumTags: 2000
# Maximum size for tag values, in bytes
maxTagValSize: 1024
# Maximum records that sync will process before returning an error
maxSyncEvents: 1000000

15
tasks/build.yml
View File

@ -16,19 +16,28 @@
ansible.builtin.command:
cmd: git submodule update --init
chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed
when: git_repository.changed
changed_when: git_repository.changed
- name: Run make setup-golpe
ansible.builtin.command:
cmd: make setup-golpe
chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed
when: git_repository.changed
changed_when: git_repository.changed
- name: Build strfry
ansible.builtin.command:
cmd: "make -j{{ strfry_make_jobs|default(1) }}"
chdir: "{{ ansible_env.HOME }}/src/strfry"
# when: git_repository.changed
when: git_repository.changed
changed_when: git_repository.changed
- name: Backup existing strfry-db
ansible.builtin.shell:
cmd: strfry export > backup.jsonl
chdir: "{{ strfry_binary_path }}"
become: yes
become_user: "{{ strfry_system_user }}"
when: git_repository.changed
changed_when: git_repository.changed

3
tasks/install.yml
View File

@ -5,6 +5,7 @@
dest: "{{ strfry_binary_path }}"
mode: 0755
remote_src: true
notify: restart strfry
- name: Setup strfry service unit
ansible.builtin.template:
@ -34,7 +35,7 @@
dest: "{{ strfry_relay.writePolicy.plugin }}"
owner: "{{ strfry_system_user }}"
group: "{{ strfry_system_group }}"
# force: false # Never overwrite, this is just a starter policy
force: false # Never overwrite, this is just a starter policy
mode: '0755'
when: strfry_policies_enabled
notify: restart strfry

66
templates/strfry.conf
View File

@ -13,6 +13,31 @@ dbParams {
# Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
mapsize = {{ strfry_dbParams.mapsize }}
# Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
noReadAhead = {{ "true" if strfry_dbParams.noReadAhead else "false" }}
}
events {
# Maximum size of normalised JSON, in bytes
maxEventSize = {{ strfry_events.maxEventSize }}
# Events newer than this will be rejected
rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }}
# Events older than this will be rejected
rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
# Maximum number of tags allowed
maxNumTags = {{ strfry_events.maxNumTags }}
# Maximum size for tag values, in bytes
maxTagValSize = {{ strfry_events.maxTagValSize }}
}
relay {
@ -63,9 +88,6 @@ relay {
writePolicy {
# If non-empty, path to an executable script that implements the writePolicy plugin logic
plugin = "{{ strfry_relay.writePolicy.plugin }}"
# Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback)
lookbackSeconds = {{ strfry_relay.writePolicy.lookbackSeconds }}
}
compression {
@ -88,6 +110,9 @@ relay {
# Log performance metrics for initial REQ database scans
dbScanPerf = {{ "true" if strfry_relay.logging.dbScanPerf else "false" }}
# Log reason for invalid event rejection? Can be disabled to silence excessive logging
invalidEvents = {{ "true" if strfry_relay.logging.invalidEvents else "false" }}
}
numThreads {
@ -100,30 +125,15 @@ relay {
# reqMonitor threads: Handle filtering of new events (restart required)
reqMonitor = {{ strfry_relay.numThreads.reqMonitor }}
# yesstr threads: Experimental yesstr protocol (restart required)
yesstr = {{ strfry_relay.numThreads.yesstr }}
# negentropy threads: Handle negentropy protocol messages (restart required)
negentropy = {{ strfry_relay.numThreads.negentropy }}
}
negentropy {
# Support negentropy protocol messages
enabled = {{ "true" if strfry_relay.negentropy.enabled else "false" }}
# Maximum records that sync will process before returning an error
maxSyncEvents = {{ strfry_relay.negentropy.maxSyncEvents }}
}
}
events {
# Maximum size of normalised JSON, in bytes
maxEventSize = {{ strfry_events.maxEventSize }}
# Events newer than this will be rejected
rejectEventsNewerThanSeconds = {{ strfry_events.rejectEventsNewerThanSeconds }}
# Events older than this will be rejected
rejectEventsOlderThanSeconds = {{ strfry_events.rejectEventsOlderThanSeconds }}
# Ephemeral events older than this will be rejected
rejectEphemeralEventsOlderThanSeconds = {{ strfry_events.rejectEphemeralEventsOlderThanSeconds }}
# Ephemeral events will be deleted from the DB when older than this
ephemeralEventsLifetimeSeconds = {{ strfry_events.ephemeralEventsLifetimeSeconds }}
# Maximum number of tags allowed
maxNumTags = {{ strfry_events.maxNumTags }}
# Maximum size for tag values, in bytes
maxTagValSize = {{ strfry_events.maxTagValSize }}
}