Merge branch 'filter' into 'develop'

Add filter task to filter jsonl events by policy

See merge request soapbox-pub/strfry-policies!2

README.md

@@ -177,6 +177,26 @@ Please look directly at `src/policies` in this repo. The files include detailed
 
 ![Policies TypeScript](https://gitlab.com/soapbox-pub/strfry-policies/uploads/dfb993b3464af5ed78bb8e5db8677458/Kazam_screencast_00090.webm)
 
+## Filtering jsonl events with your policy
+
+It is not currently possible to retroactively filter events on your strfry relay. You can however export the events with `strfry export`, filter them locally, and then import them into a fresh database. You can also use this command to filter Nostr events from any source, not just strfry.
+
+To do so, run:
+
+```sh
+cat [EVENTS_FILE] | deno task filter [POLICY_CMD] > [OUT_FILE]
+```
+
+For example:
+
+```sh
+cat events.jsonl | deno task filter ./my-policy.ts > filtered.jsonl
+```
+
+Accepted messages will be written to stdout, while rejected messages will be skipped. Also, `[POLICY_CMD]` can be _any_ strfry policy, not just one created from this repo.
+
+The command wraps each event in a strfry message of type `new`, with an `IP4` source of `127.0.0.1`, and a timestamp of the current UTC time. Therefore you may want to avoid certain policies such as the `rateLimitPolicy` that don't makes sense in this context.
+
 ## License
 
 This is free and unencumbered software released into the public domain.

deno.json

@@ -1,6 +1,7 @@
 {
   "tasks": {
-    "test": "deno test --allow-read --allow-write"
+    "test": "deno test --allow-read --allow-write",
+    "filter": "deno run -A scripts/filter.ts"
   },
   "lint": {
     "files": {

entrypoint.example.ts

@@ -16,12 +16,12 @@ import {
 for await (const msg of readStdin()) {
   const result = await pipeline(msg, [
     noopPolicy,
-    [hellthreadPolicy, { limit: 100 }],
-    [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }],
-    [rateLimitPolicy, { whitelist: ['127.0.0.1'] }],
-    [pubkeyBanPolicy, ['e810fafa1e89cdf80cced8e013938e87e21b699b24c8570537be92aec4b12c18']],
     [keywordPolicy, ['https://t.me/']],
     [regexPolicy, /(🟠|🔥|😳)ChtaGPT/i],
+    [pubkeyBanPolicy, ['e810fafa1e89cdf80cced8e013938e87e21b699b24c8570537be92aec4b12c18']],
+    [hellthreadPolicy, { limit: 100 }],
+    [rateLimitPolicy, { whitelist: ['127.0.0.1'] }],
+    [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }],
   ]);
 
   writeStdout(result);

scripts/filter.ts

@@ -0,0 +1,32 @@
+import { readLines } from '../src/deps.ts';
+
+import type { Event, InputMessage, OutputMessage } from '../mod.ts';
+
+for await (const line of readLines(Deno.stdin)) {
+  const event: Event = JSON.parse(line);
+
+  const input: InputMessage = {
+    type: 'new',
+    event: event,
+    receivedAt: Date.now() / 1000,
+    sourceType: 'IP4',
+    sourceInfo: '127.0.0.1',
+  };
+
+  const policy = Deno.run({
+    cmd: Deno.args,
+    stdin: 'piped',
+    stdout: 'piped',
+  });
+
+  await policy.stdin.write(new TextEncoder().encode(JSON.stringify(input)));
+  policy.stdin.close();
+
+  for await (const out of readLines(policy.stdout)) {
+    const msg: OutputMessage = JSON.parse(out);
+
+    if (msg.action === 'accept') {
+      console.log(JSON.stringify(event));
+    }
+  }
+}