From 8dc9f73bfd933e9cbb994922a4227d48c8c7b77f Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 28 Mar 2023 15:57:06 -0500
Subject: [PATCH 1/2] Add filter task to filter jsonl events by policy

---
 README.md             | 20 ++++++++++++++++++++
 deno.json             |  3 ++-
 entrypoint.example.ts |  0
 scripts/filter.ts     | 32 ++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 entrypoint.example.ts
 create mode 100644 scripts/filter.ts

diff --git a/README.md b/README.md
index 893745b..cb7457d 100644
--- a/README.md
+++ b/README.md
@@ -177,6 +177,26 @@ Please look directly at `src/policies` in this repo. The files include detailed
 
 ![Policies TypeScript](https://gitlab.com/soapbox-pub/strfry-policies/uploads/dfb993b3464af5ed78bb8e5db8677458/Kazam_screencast_00090.webm)
 
+## Filtering jsonl events with your policy
+
+It is not currently possible to retroactively filter events on your strfry relay. You can however export the events with `strfry export`, filter them locally, and then import them into a fresh database. You can also use this command to filter Nostr events from any source, not just strfry.
+
+To do so, run:
+
+```sh
+cat [EVENTS_FILE] | deno task filter [POLICY_CMD] > [OUT_FILE]
+```
+
+For example:
+
+```sh
+cat events.jsonl | deno task filter ./my-policy.ts > filtered.jsonl
+```
+
+Accepted messages will be written to stdout, while rejected messages will be skipped. Also, `[POLICY_CMD]` can be _any_ strfry policy, not just one created from this repo.
+
+The command wraps each event in a strfry message of type `new`, with an `IP4` source of `127.0.0.1`, and a timestamp of the current UTC time. Therefore you may want to avoid certain policies such as the `rateLimitPolicy` that don't makes sense in this context.
+
 ## License
 
 This is free and unencumbered software released into the public domain.
diff --git a/deno.json b/deno.json
index d23bcde..8aeb156 100644
--- a/deno.json
+++ b/deno.json
@@ -1,6 +1,7 @@
 {
   "tasks": {
-    "test": "deno test --allow-read --allow-write"
+    "test": "deno test --allow-read --allow-write",
+    "filter": "deno run -A scripts/filter.ts"
   },
   "lint": {
     "files": {
diff --git a/entrypoint.example.ts b/entrypoint.example.ts
old mode 100644
new mode 100755
diff --git a/scripts/filter.ts b/scripts/filter.ts
new file mode 100644
index 0000000..dc56ff6
--- /dev/null
+++ b/scripts/filter.ts
@@ -0,0 +1,32 @@
+import { readLines } from '../src/deps.ts';
+
+import type { Event, InputMessage, OutputMessage } from '../mod.ts';
+
+for await (const line of readLines(Deno.stdin)) {
+  const event: Event = JSON.parse(line);
+
+  const input: InputMessage = {
+    type: 'new',
+    event: event,
+    receivedAt: Date.now() / 1000,
+    sourceType: 'IP4',
+    sourceInfo: '127.0.0.1',
+  };
+
+  const policy = Deno.run({
+    cmd: Deno.args,
+    stdin: 'piped',
+    stdout: 'piped',
+  });
+
+  await policy.stdin.write(new TextEncoder().encode(JSON.stringify(input)));
+  policy.stdin.close();
+
+  for await (const out of readLines(policy.stdout)) {
+    const msg: OutputMessage = JSON.parse(out);
+
+    if (msg.action === 'accept') {
+      console.log(JSON.stringify(event));
+    }
+  }
+}

From 1a83ce44ac76ac2d13a4bbbf0bf2abbc43474f36 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 28 Mar 2023 16:16:47 -0500
Subject: [PATCH 2/2] Rearrange entrypoint.example.ts in a more sensible way

---
 entrypoint.example.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/entrypoint.example.ts b/entrypoint.example.ts
index 57c3087..26f1a15 100755
--- a/entrypoint.example.ts
+++ b/entrypoint.example.ts
@@ -16,12 +16,12 @@ import {
 for await (const msg of readStdin()) {
   const result = await pipeline(msg, [
     noopPolicy,
-    [hellthreadPolicy, { limit: 100 }],
-    [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }],
-    [rateLimitPolicy, { whitelist: ['127.0.0.1'] }],
-    [pubkeyBanPolicy, ['e810fafa1e89cdf80cced8e013938e87e21b699b24c8570537be92aec4b12c18']],
     [keywordPolicy, ['https://t.me/']],
     [regexPolicy, /(🟠|🔥|😳)ChtaGPT/i],
+    [pubkeyBanPolicy, ['e810fafa1e89cdf80cced8e013938e87e21b699b24c8570537be92aec4b12c18']],
+    [hellthreadPolicy, { limit: 100 }],
+    [rateLimitPolicy, { whitelist: ['127.0.0.1'] }],
+    [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }],
   ]);
 
   writeStdout(result);