From 8dc9f73bfd933e9cbb994922a4227d48c8c7b77f Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 28 Mar 2023 15:57:06 -0500
Subject: [PATCH] Add filter task to filter jsonl events by policy

---
 README.md             | 20 ++++++++++++++++++++
 deno.json             |  3 ++-
 entrypoint.example.ts |  0
 scripts/filter.ts     | 32 ++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 entrypoint.example.ts
 create mode 100644 scripts/filter.ts

diff --git a/README.md b/README.md
index 893745b..cb7457d 100644
--- a/README.md
+++ b/README.md
@@ -177,6 +177,26 @@ Please look directly at `src/policies` in this repo. The files include detailed
 
 ![Policies TypeScript](https://gitlab.com/soapbox-pub/strfry-policies/uploads/dfb993b3464af5ed78bb8e5db8677458/Kazam_screencast_00090.webm)
 
+## Filtering jsonl events with your policy
+
+It is not currently possible to retroactively filter events on your strfry relay. You can however export the events with `strfry export`, filter them locally, and then import them into a fresh database. You can also use this command to filter Nostr events from any source, not just strfry.
+
+To do so, run:
+
+```sh
+cat [EVENTS_FILE] | deno task filter [POLICY_CMD] > [OUT_FILE]
+```
+
+For example:
+
+```sh
+cat events.jsonl | deno task filter ./my-policy.ts > filtered.jsonl
+```
+
+Accepted messages will be written to stdout, while rejected messages will be skipped. Also, `[POLICY_CMD]` can be _any_ strfry policy, not just one created from this repo.
+
+The command wraps each event in a strfry message of type `new`, with an `IP4` source of `127.0.0.1`, and a timestamp of the current UTC time. Therefore you may want to avoid certain policies such as the `rateLimitPolicy` that don't makes sense in this context.
+
 ## License
 
 This is free and unencumbered software released into the public domain.
diff --git a/deno.json b/deno.json
index d23bcde..8aeb156 100644
--- a/deno.json
+++ b/deno.json
@@ -1,6 +1,7 @@
 {
   "tasks": {
-    "test": "deno test --allow-read --allow-write"
+    "test": "deno test --allow-read --allow-write",
+    "filter": "deno run -A scripts/filter.ts"
   },
   "lint": {
     "files": {
diff --git a/entrypoint.example.ts b/entrypoint.example.ts
old mode 100644
new mode 100755
diff --git a/scripts/filter.ts b/scripts/filter.ts
new file mode 100644
index 0000000..dc56ff6
--- /dev/null
+++ b/scripts/filter.ts
@@ -0,0 +1,32 @@
+import { readLines } from '../src/deps.ts';
+
+import type { Event, InputMessage, OutputMessage } from '../mod.ts';
+
+for await (const line of readLines(Deno.stdin)) {
+  const event: Event = JSON.parse(line);
+
+  const input: InputMessage = {
+    type: 'new',
+    event: event,
+    receivedAt: Date.now() / 1000,
+    sourceType: 'IP4',
+    sourceInfo: '127.0.0.1',
+  };
+
+  const policy = Deno.run({
+    cmd: Deno.args,
+    stdin: 'piped',
+    stdout: 'piped',
+  });
+
+  await policy.stdin.write(new TextEncoder().encode(JSON.stringify(input)));
+  policy.stdin.close();
+
+  for await (const out of readLines(policy.stdout)) {
+    const msg: OutputMessage = JSON.parse(out);
+
+    if (msg.action === 'accept') {
+      console.log(JSON.stringify(event));
+    }
+  }
+}