From 7e503a3892927b4637c9a15e7a19d8b89bc6befd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 13:06:56 -0500 Subject: [PATCH 01/17] Move policies into subdirectory --- .../policies/anti-duplication-policy.ts | 0 hellthread-policy.ts => src/policies/hellthread-policy.ts | 0 noop-policy.ts => src/policies/noop-policy.ts | 0 rate-limit-policy.ts => src/policies/rate-limit-policy.ts | 0 read-only-policy.ts => src/policies/read-only-policy.ts | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename anti-duplication-policy.ts => src/policies/anti-duplication-policy.ts (100%) rename hellthread-policy.ts => src/policies/hellthread-policy.ts (100%) rename noop-policy.ts => src/policies/noop-policy.ts (100%) rename rate-limit-policy.ts => src/policies/rate-limit-policy.ts (100%) rename read-only-policy.ts => src/policies/read-only-policy.ts (100%) diff --git a/anti-duplication-policy.ts b/src/policies/anti-duplication-policy.ts similarity index 100% rename from anti-duplication-policy.ts rename to src/policies/anti-duplication-policy.ts diff --git a/hellthread-policy.ts b/src/policies/hellthread-policy.ts similarity index 100% rename from hellthread-policy.ts rename to src/policies/hellthread-policy.ts diff --git a/noop-policy.ts b/src/policies/noop-policy.ts similarity index 100% rename from noop-policy.ts rename to src/policies/noop-policy.ts diff --git a/rate-limit-policy.ts b/src/policies/rate-limit-policy.ts similarity index 100% rename from rate-limit-policy.ts rename to src/policies/rate-limit-policy.ts diff --git a/read-only-policy.ts b/src/policies/read-only-policy.ts similarity index 100% rename from read-only-policy.ts rename to src/policies/read-only-policy.ts From e3df8579a06a52ae15a44c23771c980fc3dce046 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 13:19:35 -0500 Subject: [PATCH 02/17] Move types into their own file, import them --- src/policies/anti-duplication-policy.ts | 26 ++--------------------- src/policies/hellthread-policy.ts | 26 ++--------------------- src/policies/noop-policy.ts | 24 +-------------------- src/policies/rate-limit-policy.ts | 28 +++---------------------- src/policies/read-only-policy.ts | 24 +-------------------- src/types.ts | 25 ++++++++++++++++++++++ 6 files changed, 34 insertions(+), 119 deletions(-) create mode 100644 src/types.ts diff --git a/src/policies/anti-duplication-policy.ts b/src/policies/anti-duplication-policy.ts index c51c3e0..7bb032f 100755 --- a/src/policies/anti-duplication-policy.ts +++ b/src/policies/anti-duplication-policy.ts @@ -3,33 +3,11 @@ import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; import { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; +import type { InputMessage, OutputMessage } from '../types.ts'; + const ANTI_DUPLICATION_TTL = Number(Deno.env.get('ANTI_DUPLICATION_TTL') || 60000); const ANTI_DUPLICATION_MIN_LENGTH = Number(Deno.env.get('ANTI_DUPLICATION_MIN_LENGTH') || 50); -interface InputMessage { - type: 'new' | 'lookback'; - event: Event; - receivedAt: number; - sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; - sourceInfo: string; -} - -interface OutputMessage { - id: string; - action: 'accept' | 'reject' | 'shadowReject'; - msg: string; -} - -interface Event { - id: string; - sig: string; - kind: number; - tags: string[][]; - pubkey: string; - content: string; - created_at: number; -} - /** https://stackoverflow.com/a/8831937 */ function hashCode(str: string): number { let hash = 0; diff --git a/src/policies/hellthread-policy.ts b/src/policies/hellthread-policy.ts index d89f9c9..2120dbf 100755 --- a/src/policies/hellthread-policy.ts +++ b/src/policies/hellthread-policy.ts @@ -1,32 +1,10 @@ #!/usr/bin/env -S deno run import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +import type { InputMessage, OutputMessage } from '../types.ts'; + const HELLTHREAD_LIMIT = Number(Deno.env.get('HELLTHREAD_LIMIT') || 100); -interface InputMessage { - type: 'new' | 'lookback'; - event: Event; - receivedAt: number; - sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; - sourceInfo: string; -} - -interface OutputMessage { - id: string; - action: 'accept' | 'reject' | 'shadowReject'; - msg: string; -} - -interface Event { - id: string; - sig: string; - kind: number; - tags: string[][]; - pubkey: string; - content: string; - created_at: number; -} - function handleMessage(msg: InputMessage): OutputMessage { if (msg.event.kind === 1) { const p = msg.event.tags.filter((tag) => tag[0] === 'p'); diff --git a/src/policies/noop-policy.ts b/src/policies/noop-policy.ts index e3361df..912ed46 100755 --- a/src/policies/noop-policy.ts +++ b/src/policies/noop-policy.ts @@ -1,29 +1,7 @@ #!/usr/bin/env -S deno run import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; -interface InputMessage { - type: 'new' | 'lookback'; - event: Event; - receivedAt: number; - sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; - sourceInfo: string; -} - -interface OutputMessage { - id: string; - action: 'accept' | 'reject' | 'shadowReject'; - msg: string; -} - -interface Event { - id: string; - sig: string; - kind: number; - tags: string[][]; - pubkey: string; - content: string; - created_at: number; -} +import type { InputMessage, OutputMessage } from '../types.ts'; function handleMessage(msg: InputMessage): OutputMessage { return { diff --git a/src/policies/rate-limit-policy.ts b/src/policies/rate-limit-policy.ts index 54da6ec..1e14f2f 100755 --- a/src/policies/rate-limit-policy.ts +++ b/src/policies/rate-limit-policy.ts @@ -3,41 +3,19 @@ import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; import { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; +import type { InputMessage, OutputMessage } from '../types.ts'; + const IP_WHITELIST = (Deno.env.get('IP_WHITELIST') || '').split(','); const RATE_LIMIT_INTERVAL = Number(Deno.env.get('RATE_LIMIT_INTERVAL') || 60000); const RATE_LIMIT_MAX = Number(Deno.env.get('RATE_LIMIT_MAX') || 10); -interface InputMessage { - type: 'new' | 'lookback'; - event: Event; - receivedAt: number; - sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; - sourceInfo: string; -} - -interface OutputMessage { - id: string; - action: 'accept' | 'reject' | 'shadowReject'; - msg: string; -} - -interface Event { - id: string; - sig: string; - kind: number; - tags: string[][]; - pubkey: string; - content: string; - created_at: number; -} - async function handleMessage(msg: InputMessage): Promise { if ((msg.sourceType === 'IP4' || msg.sourceType === 'IP6') && !IP_WHITELIST.includes(msg.sourceInfo)) { const db = new Keydb('sqlite:///tmp/strfry-rate-limit-policy.sqlite3'); const count = await db.get(msg.sourceInfo) || 0; await db.set(msg.sourceInfo, count + 1, RATE_LIMIT_INTERVAL); - + if (count >= RATE_LIMIT_MAX) { return { id: msg.event.id, diff --git a/src/policies/read-only-policy.ts b/src/policies/read-only-policy.ts index f1e11af..998dcc6 100755 --- a/src/policies/read-only-policy.ts +++ b/src/policies/read-only-policy.ts @@ -2,29 +2,7 @@ //bin/true; exec deno run -A "$0" "$@" import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; -interface InputMessage { - type: 'new' | 'lookback'; - event: Event; - receivedAt: number; - sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; - sourceInfo: string; -} - -interface OutputMessage { - id: string; - action: 'accept' | 'reject' | 'shadowReject'; - msg: string; -} - -interface Event { - id: string; - sig: string; - kind: number; - tags: string[][]; - pubkey: string; - content: string; - created_at: number; -} +import type { InputMessage, OutputMessage } from '../types.ts'; function handleMessage(msg: InputMessage): OutputMessage { return { diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..473a3f7 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,25 @@ +interface InputMessage { + type: 'new' | 'lookback'; + event: Event; + receivedAt: number; + sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; + sourceInfo: string; +} + +interface OutputMessage { + id: string; + action: 'accept' | 'reject' | 'shadowReject'; + msg: string; +} + +interface Event { + id: string; + sig: string; + kind: K; + tags: string[][]; + pubkey: string; + content: string; + created_at: number; +} + +export type { Event, InputMessage, OutputMessage }; From 64413d572ba7d3b03bf8d18ed2509b3c2254aec0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 13:23:02 -0500 Subject: [PATCH 03/17] Add deps.ts --- src/deps.ts | 2 ++ src/policies/anti-duplication-policy.ts | 3 +-- src/policies/hellthread-policy.ts | 2 +- src/policies/noop-policy.ts | 2 +- src/policies/rate-limit-policy.ts | 3 +-- src/policies/read-only-policy.ts | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 src/deps.ts diff --git a/src/deps.ts b/src/deps.ts new file mode 100644 index 0000000..0544dde --- /dev/null +++ b/src/deps.ts @@ -0,0 +1,2 @@ +export { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +export { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; diff --git a/src/policies/anti-duplication-policy.ts b/src/policies/anti-duplication-policy.ts index 7bb032f..a943749 100755 --- a/src/policies/anti-duplication-policy.ts +++ b/src/policies/anti-duplication-policy.ts @@ -1,7 +1,6 @@ #!/bin/sh //bin/true; exec deno run -A "$0" "$@" -import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; -import { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; +import { Keydb, readLines } from '../deps.ts'; import type { InputMessage, OutputMessage } from '../types.ts'; diff --git a/src/policies/hellthread-policy.ts b/src/policies/hellthread-policy.ts index 2120dbf..6842176 100755 --- a/src/policies/hellthread-policy.ts +++ b/src/policies/hellthread-policy.ts @@ -1,5 +1,5 @@ #!/usr/bin/env -S deno run -import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +import { readLines } from '../deps.ts'; import type { InputMessage, OutputMessage } from '../types.ts'; diff --git a/src/policies/noop-policy.ts b/src/policies/noop-policy.ts index 912ed46..1942e3a 100755 --- a/src/policies/noop-policy.ts +++ b/src/policies/noop-policy.ts @@ -1,5 +1,5 @@ #!/usr/bin/env -S deno run -import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +import { readLines } from '../deps.ts'; import type { InputMessage, OutputMessage } from '../types.ts'; diff --git a/src/policies/rate-limit-policy.ts b/src/policies/rate-limit-policy.ts index 1e14f2f..906f242 100755 --- a/src/policies/rate-limit-policy.ts +++ b/src/policies/rate-limit-policy.ts @@ -1,7 +1,6 @@ #!/bin/sh //bin/true; exec deno run -A "$0" "$@" -import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; -import { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; +import { Keydb, readLines } from '../deps.ts'; import type { InputMessage, OutputMessage } from '../types.ts'; diff --git a/src/policies/read-only-policy.ts b/src/policies/read-only-policy.ts index 998dcc6..881a9a8 100755 --- a/src/policies/read-only-policy.ts +++ b/src/policies/read-only-policy.ts @@ -1,6 +1,6 @@ #!/bin/sh //bin/true; exec deno run -A "$0" "$@" -import { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +import { readLines } from '../deps.ts'; import type { InputMessage, OutputMessage } from '../types.ts'; From 78fae382684c1c7eba9748804d3f1778882da828 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 14:36:11 -0500 Subject: [PATCH 04/17] Rewrite all policies as policy modules --- entrypoint.example.ts | 4 +++ src/pipeline.ts | 3 ++ src/policies/anti-duplication-policy.ts | 43 ++++++++++++++----------- src/policies/hellthread-policy.ts | 14 +++----- src/policies/noop-policy.ts | 25 +++++++------- src/policies/rate-limit-policy.ts | 19 +++++------ src/policies/read-only-policy.ts | 23 +++++-------- src/stdin.ts | 14 ++++++++ src/types.ts | 4 ++- 9 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 entrypoint.example.ts create mode 100644 src/pipeline.ts create mode 100644 src/stdin.ts diff --git a/entrypoint.example.ts b/entrypoint.example.ts new file mode 100644 index 0000000..0cd6f3b --- /dev/null +++ b/entrypoint.example.ts @@ -0,0 +1,4 @@ +#!/bin/sh +//bin/true; exec deno run -A "$0" "$@" + +// TODO diff --git a/src/pipeline.ts b/src/pipeline.ts new file mode 100644 index 0000000..df068ab --- /dev/null +++ b/src/pipeline.ts @@ -0,0 +1,3 @@ +import { readStdin } from './stdin.ts'; + +console.log(await readStdin()); diff --git a/src/policies/anti-duplication-policy.ts b/src/policies/anti-duplication-policy.ts index a943749..a9f15bf 100755 --- a/src/policies/anti-duplication-policy.ts +++ b/src/policies/anti-duplication-policy.ts @@ -1,24 +1,16 @@ -#!/bin/sh -//bin/true; exec deno run -A "$0" "$@" -import { Keydb, readLines } from '../deps.ts'; +import { Keydb } from '../deps.ts'; -import type { InputMessage, OutputMessage } from '../types.ts'; +import type { Policy } from '../types.ts'; const ANTI_DUPLICATION_TTL = Number(Deno.env.get('ANTI_DUPLICATION_TTL') || 60000); const ANTI_DUPLICATION_MIN_LENGTH = Number(Deno.env.get('ANTI_DUPLICATION_MIN_LENGTH') || 50); -/** https://stackoverflow.com/a/8831937 */ -function hashCode(str: string): number { - let hash = 0; - for (let i = 0, len = str.length; i < len; i++) { - const chr = str.charCodeAt(i); - hash = (hash << 5) - hash + chr; - hash |= 0; // Convert to 32bit integer - } - return hash; -} - -async function handleMessage(msg: InputMessage): Promise { +/** + * Prevent messages with the exact same content from being submitted repeatedly. + * It stores a hashcode for each content in an SQLite database and rate-limits them. + * Only messages that meet the minimum length criteria are selected. + */ +const antiDuplicationPolicy: Policy = async (msg) => { const { kind, content } = msg.event; if (kind === 1 && content.length >= ANTI_DUPLICATION_MIN_LENGTH) { @@ -42,8 +34,21 @@ async function handleMessage(msg: InputMessage): Promise { action: 'accept', msg: '', }; +}; + +/** + * Get a "good enough" unique identifier for this content. + * This algorithm was chosen because it's very fast with a low chance of collisions. + * https://stackoverflow.com/a/8831937 + */ +function hashCode(str: string): number { + let hash = 0; + for (let i = 0, len = str.length; i < len; i++) { + const chr = str.charCodeAt(i); + hash = (hash << 5) - hash + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; } -for await (const line of readLines(Deno.stdin)) { - console.log(JSON.stringify(await handleMessage(JSON.parse(line)))); -} +export default antiDuplicationPolicy; diff --git a/src/policies/hellthread-policy.ts b/src/policies/hellthread-policy.ts index 6842176..419415b 100755 --- a/src/policies/hellthread-policy.ts +++ b/src/policies/hellthread-policy.ts @@ -1,11 +1,9 @@ -#!/usr/bin/env -S deno run -import { readLines } from '../deps.ts'; - -import type { InputMessage, OutputMessage } from '../types.ts'; +import type { Policy } from '../types.ts'; const HELLTHREAD_LIMIT = Number(Deno.env.get('HELLTHREAD_LIMIT') || 100); -function handleMessage(msg: InputMessage): OutputMessage { +/** Reject messages that tag too many participants. */ +const hellthreadPolicy: Policy = (msg) => { if (msg.event.kind === 1) { const p = msg.event.tags.filter((tag) => tag[0] === 'p'); @@ -23,8 +21,6 @@ function handleMessage(msg: InputMessage): OutputMessage { action: 'accept', msg: '', }; -} +}; -for await (const line of readLines(Deno.stdin)) { - console.log(JSON.stringify(handleMessage(JSON.parse(line)))); -} +export default hellthreadPolicy; diff --git a/src/policies/noop-policy.ts b/src/policies/noop-policy.ts index 1942e3a..a930ef9 100755 --- a/src/policies/noop-policy.ts +++ b/src/policies/noop-policy.ts @@ -1,16 +1,13 @@ -#!/usr/bin/env -S deno run -import { readLines } from '../deps.ts'; +import type { Policy } from '../types.ts'; -import type { InputMessage, OutputMessage } from '../types.ts'; +/** + * Minimal sample policy for demonstration purposes. + * Allows all events through. + */ +const noopPolicy: Policy = (msg) => ({ + id: msg.event.id, + action: 'accept', + msg: '', +}); -function handleMessage(msg: InputMessage): OutputMessage { - return { - id: msg.event.id, - action: 'accept', - msg: '', - }; -} - -for await (const line of readLines(Deno.stdin)) { - console.log(JSON.stringify(handleMessage(JSON.parse(line)))); -} +export default noopPolicy; diff --git a/src/policies/rate-limit-policy.ts b/src/policies/rate-limit-policy.ts index 906f242..a6810bb 100755 --- a/src/policies/rate-limit-policy.ts +++ b/src/policies/rate-limit-policy.ts @@ -1,15 +1,18 @@ -#!/bin/sh -//bin/true; exec deno run -A "$0" "$@" -import { Keydb, readLines } from '../deps.ts'; +import { Keydb } from '../deps.ts'; -import type { InputMessage, OutputMessage } from '../types.ts'; +import type { Policy } from '../types.ts'; const IP_WHITELIST = (Deno.env.get('IP_WHITELIST') || '').split(','); const RATE_LIMIT_INTERVAL = Number(Deno.env.get('RATE_LIMIT_INTERVAL') || 60000); const RATE_LIMIT_MAX = Number(Deno.env.get('RATE_LIMIT_MAX') || 10); -async function handleMessage(msg: InputMessage): Promise { +/** + * Rate-limits users by their IP address. + * IPs are stored in an SQLite database. If you are running internal services, + * it's a good idea to at least whitelist `127.0.0.1` etc. + */ +const rateLimitPolicy: Policy = async (msg) => { if ((msg.sourceType === 'IP4' || msg.sourceType === 'IP6') && !IP_WHITELIST.includes(msg.sourceInfo)) { const db = new Keydb('sqlite:///tmp/strfry-rate-limit-policy.sqlite3'); const count = await db.get(msg.sourceInfo) || 0; @@ -29,8 +32,6 @@ async function handleMessage(msg: InputMessage): Promise { action: 'accept', msg: '', }; -} +}; -for await (const line of readLines(Deno.stdin)) { - console.log(JSON.stringify(await handleMessage(JSON.parse(line)))); -} +export default rateLimitPolicy; diff --git a/src/policies/read-only-policy.ts b/src/policies/read-only-policy.ts index 881a9a8..996c27b 100755 --- a/src/policies/read-only-policy.ts +++ b/src/policies/read-only-policy.ts @@ -1,17 +1,10 @@ -#!/bin/sh -//bin/true; exec deno run -A "$0" "$@" -import { readLines } from '../deps.ts'; +import type { Policy } from '../types.ts'; -import type { InputMessage, OutputMessage } from '../types.ts'; +/** This policy rejects all messages. */ +const readOnlyPolicy: Policy = (msg) => ({ + id: msg.event.id, + action: 'reject', + msg: 'The relay is read-only.', +}); -function handleMessage(msg: InputMessage): OutputMessage { - return { - id: msg.event.id, - action: 'reject', - msg: 'The relay is set to read-only.', - }; -} - -for await (const line of readLines(Deno.stdin)) { - console.log(JSON.stringify(handleMessage(JSON.parse(line)))); -} +export default readOnlyPolicy; diff --git a/src/stdin.ts b/src/stdin.ts new file mode 100644 index 0000000..3305d19 --- /dev/null +++ b/src/stdin.ts @@ -0,0 +1,14 @@ +import { readLines } from './deps.ts'; + +import type { InputMessage } from './types.ts'; + +/** + * Get the first line from stdin. + * Can only be read ONCE, or else it returns undefined. + */ +async function readStdin(): Promise { + const { value } = await readLines(Deno.stdin).next(); + return JSON.parse(value); +} + +export { readStdin }; diff --git a/src/types.ts b/src/types.ts index 473a3f7..8f3f4a6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,4 +22,6 @@ interface Event { created_at: number; } -export type { Event, InputMessage, OutputMessage }; +type Policy = (msg: InputMessage) => Promise | OutputMessage; + +export type { Event, InputMessage, OutputMessage, Policy }; From 8e41f410020056fda20b2e965485b7d506bd26d8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 14:42:12 -0500 Subject: [PATCH 05/17] Add mod.ts --- mod.ts | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 mod.ts diff --git a/mod.ts b/mod.ts new file mode 100644 index 0000000..bebdbdd --- /dev/null +++ b/mod.ts @@ -0,0 +1,5 @@ +export { default as antiDuplicationPolicy } from './src/policies/anti-duplication-policy.ts'; +export { default as hellthreadPolicy } from './src/policies/hellthread-policy.ts'; +export { default as noopPolicy } from './src/policies/noop-policy.ts'; +export { default as rateLimitPolicy } from './src/policies/rate-limit-policy.ts'; +export { default as readOnlyPolicy } from './src/policies/read-only-policy.ts'; From b5fe0c5e673eafee67d51e007fe4f5b5219c76f2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 16:08:36 -0500 Subject: [PATCH 06/17] Add pipeline, scope out entrypoint.example.ts --- entrypoint.example.ts | 20 +++++++++++++++++++- mod.ts | 3 +++ src/{stdin.ts => io.ts} | 9 +++++++-- src/pipeline.ts | 20 ++++++++++++++++++-- 4 files changed, 47 insertions(+), 5 deletions(-) rename src/{stdin.ts => io.ts} (54%) diff --git a/entrypoint.example.ts b/entrypoint.example.ts index 0cd6f3b..5898a94 100644 --- a/entrypoint.example.ts +++ b/entrypoint.example.ts @@ -1,4 +1,22 @@ #!/bin/sh //bin/true; exec deno run -A "$0" "$@" +import { + antiDuplicationPolicy, + hellthreadPolicy, + noopPolicy, + pipeline, + rateLimitPolicy, + readStdin, + writeStdout, +} from './mod.ts'; -// TODO +const msg = await readStdin(); + +const result = await pipeline(msg, [ + noopPolicy, + hellthreadPolicy, + antiDuplicationPolicy, + rateLimitPolicy, +]); + +writeStdout(result); diff --git a/mod.ts b/mod.ts index bebdbdd..e00c0d2 100644 --- a/mod.ts +++ b/mod.ts @@ -3,3 +3,6 @@ export { default as hellthreadPolicy } from './src/policies/hellthread-policy.ts export { default as noopPolicy } from './src/policies/noop-policy.ts'; export { default as rateLimitPolicy } from './src/policies/rate-limit-policy.ts'; export { default as readOnlyPolicy } from './src/policies/read-only-policy.ts'; + +export { readStdin, writeStdout } from './src/io.ts'; +export { default as pipeline } from './src/pipeline.ts'; diff --git a/src/stdin.ts b/src/io.ts similarity index 54% rename from src/stdin.ts rename to src/io.ts index 3305d19..d62aa8c 100644 --- a/src/stdin.ts +++ b/src/io.ts @@ -1,6 +1,6 @@ import { readLines } from './deps.ts'; -import type { InputMessage } from './types.ts'; +import type { InputMessage, OutputMessage } from './types.ts'; /** * Get the first line from stdin. @@ -11,4 +11,9 @@ async function readStdin(): Promise { return JSON.parse(value); } -export { readStdin }; +/** Writes the output message to stdout. */ +function writeStdout(msg: OutputMessage): void { + console.log(JSON.stringify(msg)); +} + +export { readStdin, writeStdout }; diff --git a/src/pipeline.ts b/src/pipeline.ts index df068ab..70a0260 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,3 +1,19 @@ -import { readStdin } from './stdin.ts'; +import { InputMessage, OutputMessage, Policy } from './types.ts'; -console.log(await readStdin()); +/** Processes messages through multiple policies, bailing early on rejection. */ +async function pipeline(msg: InputMessage, policies: Policy[]): Promise { + for (const policy of policies) { + const result = await policy(msg); + if (result.action !== 'accept') { + return result; + } + } + + return { + id: msg.event.id, + action: 'accept', + msg: '', + }; +} + +export default pipeline; From 2d7d2da964f2d301da4dbd9fbc40f4823a62a0eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 19:55:58 -0500 Subject: [PATCH 07/17] Make pipeline take tuples with opts --- entrypoint.example.ts | 8 ++++---- src/pipeline.ts | 13 ++++++++++--- src/policies/hellthread-policy.ts | 12 ++++++++---- src/types.ts | 2 +- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/entrypoint.example.ts b/entrypoint.example.ts index 5898a94..335e1f8 100644 --- a/entrypoint.example.ts +++ b/entrypoint.example.ts @@ -13,10 +13,10 @@ import { const msg = await readStdin(); const result = await pipeline(msg, [ - noopPolicy, - hellthreadPolicy, - antiDuplicationPolicy, - rateLimitPolicy, + [noopPolicy], + [hellthreadPolicy, { limit: 100 }], + [antiDuplicationPolicy], + [rateLimitPolicy], ]); writeStdout(result); diff --git a/src/pipeline.ts b/src/pipeline.ts index 70a0260..7d8e6c3 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,9 +1,16 @@ import { InputMessage, OutputMessage, Policy } from './types.ts'; +type PolicyTuple = [policy: Policy, opts?: Opts]; + +type PolicyTuplesRest = { + [K in keyof T]: PolicyTuple +} + /** Processes messages through multiple policies, bailing early on rejection. */ -async function pipeline(msg: InputMessage, policies: Policy[]): Promise { - for (const policy of policies) { - const result = await policy(msg); +async function pipeline

(msg: InputMessage, policies: [...PolicyTuplesRest

]): Promise { + for (const tuple of policies) { + const [policy, opts] = tuple; + const result = await policy(msg, opts); if (result.action !== 'accept') { return result; } diff --git a/src/policies/hellthread-policy.ts b/src/policies/hellthread-policy.ts index 419415b..bb980ee 100755 --- a/src/policies/hellthread-policy.ts +++ b/src/policies/hellthread-policy.ts @@ -1,17 +1,21 @@ import type { Policy } from '../types.ts'; -const HELLTHREAD_LIMIT = Number(Deno.env.get('HELLTHREAD_LIMIT') || 100); +interface Hellthread { + limit: number; +} /** Reject messages that tag too many participants. */ -const hellthreadPolicy: Policy = (msg) => { +const hellthreadPolicy: Policy = (msg, opts) => { + const limit = opts?.limit || 100; + if (msg.event.kind === 1) { const p = msg.event.tags.filter((tag) => tag[0] === 'p'); - if (p.length > HELLTHREAD_LIMIT) { + if (p.length > limit) { return { id: msg.event.id, action: 'reject', - msg: `Event rejected due to ${p.length} "p" tags (${HELLTHREAD_LIMIT} is the limit).`, + msg: `Event rejected due to ${p.length} "p" tags (${limit} is the limit).`, }; } } diff --git a/src/types.ts b/src/types.ts index 8f3f4a6..6243eaa 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,6 +22,6 @@ interface Event { created_at: number; } -type Policy = (msg: InputMessage) => Promise | OutputMessage; +type Policy = (msg: InputMessage, opts?: Opts) => Promise | OutputMessage; export type { Event, InputMessage, OutputMessage, Policy }; From f2f4dd7b8e7384803cbca6d4ebf7b9c299909153 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:06:03 -0500 Subject: [PATCH 08/17] Allow non-tuple values --- entrypoint.example.ts | 6 +++--- src/pipeline.ts | 15 +++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/entrypoint.example.ts b/entrypoint.example.ts index 335e1f8..8ec5df6 100644 --- a/entrypoint.example.ts +++ b/entrypoint.example.ts @@ -13,10 +13,10 @@ import { const msg = await readStdin(); const result = await pipeline(msg, [ - [noopPolicy], + noopPolicy, [hellthreadPolicy, { limit: 100 }], - [antiDuplicationPolicy], - [rateLimitPolicy], + antiDuplicationPolicy, + rateLimitPolicy, ]); writeStdout(result); diff --git a/src/pipeline.ts b/src/pipeline.ts index 7d8e6c3..7297844 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,15 +1,17 @@ import { InputMessage, OutputMessage, Policy } from './types.ts'; -type PolicyTuple = [policy: Policy, opts?: Opts]; +type PolicyTuple = [policy: Policy, opts?: Opts]; +// https://stackoverflow.com/a/75806165 +// https://stackoverflow.com/a/54608401 type PolicyTuplesRest = { - [K in keyof T]: PolicyTuple + [K in keyof T]: PolicyTuple | Policy } /** Processes messages through multiple policies, bailing early on rejection. */ async function pipeline

(msg: InputMessage, policies: [...PolicyTuplesRest

]): Promise { - for (const tuple of policies) { - const [policy, opts] = tuple; + for (const item of policies) { + const [policy, opts] = toTuple(item); const result = await policy(msg, opts); if (result.action !== 'accept') { return result; @@ -23,4 +25,9 @@ async function pipeline

(msg: InputMessage, policies: [...Policy }; } +/** Coerce item into a tuple if it isn't already. */ +function toTuple(item: PolicyTuple | Policy): PolicyTuple { + return typeof item === 'function' ? [item] : item; +} + export default pipeline; From 5a6af744132415c4905fbf082936c686465209e3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:17:51 -0500 Subject: [PATCH 09/17] Add comments to types, deno fmt --- src/pipeline.ts | 11 ++++++++--- src/types.ts | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 7297844..3c5d1dc 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,15 +1,20 @@ import { InputMessage, OutputMessage, Policy } from './types.ts'; +/** A policy function with opts to run it with. Used by the pipeline. */ type PolicyTuple = [policy: Policy, opts?: Opts]; +/** Helper type for proper type inference of PolicyTuples in the pipeline. */ // https://stackoverflow.com/a/75806165 // https://stackoverflow.com/a/54608401 type PolicyTuplesRest = { - [K in keyof T]: PolicyTuple | Policy -} + [K in keyof T]: PolicyTuple | Policy; +}; /** Processes messages through multiple policies, bailing early on rejection. */ -async function pipeline

(msg: InputMessage, policies: [...PolicyTuplesRest

]): Promise { +async function pipeline

( + msg: InputMessage, + policies: [...PolicyTuplesRest

], +): Promise { for (const item of policies) { const [policy, opts] = toTuple(item); const result = await policy(msg, opts); diff --git a/src/types.ts b/src/types.ts index 6243eaa..f5e45c5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,17 +1,37 @@ +/** + * strfry input message from stdin. + * https://github.com/hoytech/strfry/blob/master/docs/plugins.md#input-messages + */ interface InputMessage { + /** Either `new` or `lookback`. */ type: 'new' | 'lookback'; + /** The event posted by the client, with all the required fields such as `id`, `pubkey`, etc. */ event: Event; + /** Unix timestamp of when this event was received by the relay. */ receivedAt: number; + /** Where this event came from. Typically will be `IP4` or `IP6`, but in lookback can also be `Import`, `Stream`, or `Sync`. */ sourceType: 'IP4' | 'IP6' | 'Import' | 'Stream' | 'Sync'; + /** Specifics of the event's source. Either an IP address or a relay URL (for stream/sync). */ sourceInfo: string; } +/** + * strfry output message to be printed as JSONL (minified JSON followed by a newline) to stdout. + * https://github.com/hoytech/strfry/blob/master/docs/plugins.md#output-messages + */ interface OutputMessage { + /** The event ID taken from the `event.id` field of the input message. */ id: string; + /** Either `accept`, `reject`, or `shadowReject`. */ action: 'accept' | 'reject' | 'shadowReject'; + /** The NIP-20 response message to be sent to the client. Only used for `reject`. */ msg: string; } +/** + * Nostr event. + * https://github.com/nostr-protocol/nips/blob/master/01.md + */ interface Event { id: string; sig: string; @@ -22,6 +42,10 @@ interface Event { created_at: number; } +/** + * A policy function in this library. + * It accepts an input message, opts, and returns an output message. + */ type Policy = (msg: InputMessage, opts?: Opts) => Promise | OutputMessage; export type { Event, InputMessage, OutputMessage, Policy }; From 6e342b9667e8f3405fd8cc72b377a5794939dcbb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:23:38 -0500 Subject: [PATCH 10/17] Tweak types a little --- src/pipeline.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 3c5d1dc..9d379a4 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -6,15 +6,12 @@ type PolicyTuple = [policy: Policy, opts?: Opts]; /** Helper type for proper type inference of PolicyTuples in the pipeline. */ // https://stackoverflow.com/a/75806165 // https://stackoverflow.com/a/54608401 -type PolicyTuplesRest = { +type PolicySpread = { [K in keyof T]: PolicyTuple | Policy; }; /** Processes messages through multiple policies, bailing early on rejection. */ -async function pipeline

( - msg: InputMessage, - policies: [...PolicyTuplesRest

], -): Promise { +async function pipeline(msg: InputMessage, policies: [...PolicySpread]): Promise { for (const item of policies) { const [policy, opts] = toTuple(item); const result = await policy(msg, opts); From 2f4ebfbda2e252e61d23c255a7089e974d4c2d14 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:27:07 -0500 Subject: [PATCH 11/17] Export useful types in mod.ts --- mod.ts | 3 +++ src/pipeline.ts | 2 ++ 2 files changed, 5 insertions(+) diff --git a/mod.ts b/mod.ts index e00c0d2..d05e972 100644 --- a/mod.ts +++ b/mod.ts @@ -6,3 +6,6 @@ export { default as readOnlyPolicy } from './src/policies/read-only-policy.ts'; export { readStdin, writeStdout } from './src/io.ts'; export { default as pipeline } from './src/pipeline.ts'; + +export type { Event, InputMessage, OutputMessage, Policy } from './src/types.ts'; +export type { PolicyTuple } from './src/pipeline.ts'; diff --git a/src/pipeline.ts b/src/pipeline.ts index 9d379a4..ff0469e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -33,3 +33,5 @@ function toTuple(item: PolicyTuple | Policy): PolicyTuple { } export default pipeline; + +export type { PolicyTuple }; From ae0242fc269fd60da39c96759bee90b75b227b5c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:48:55 -0500 Subject: [PATCH 12/17] Make all policies accept opts, get rid of envvars --- src/policies/anti-duplication-policy.ts | 26 +++++++++++++------ src/policies/hellthread-policy.ts | 7 ++++-- src/policies/noop-policy.ts | 2 +- src/policies/rate-limit-policy.ts | 33 +++++++++++++++++-------- src/policies/read-only-policy.ts | 2 +- 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/policies/anti-duplication-policy.ts b/src/policies/anti-duplication-policy.ts index a9f15bf..7db9701 100755 --- a/src/policies/anti-duplication-policy.ts +++ b/src/policies/anti-duplication-policy.ts @@ -2,23 +2,33 @@ import { Keydb } from '../deps.ts'; import type { Policy } from '../types.ts'; -const ANTI_DUPLICATION_TTL = Number(Deno.env.get('ANTI_DUPLICATION_TTL') || 60000); -const ANTI_DUPLICATION_MIN_LENGTH = Number(Deno.env.get('ANTI_DUPLICATION_MIN_LENGTH') || 50); +interface AntiDuplication { + /** Time in ms until a message with this content may be posted again. Default: `60000` (1 minute). */ + ttl?: number; + /** Note text under this limit will be skipped by the policy. Default: `50`. */ + minLength?: number; + /** Database connection string. Default: `sqlite:///tmp/strfry-anti-duplication-policy.sqlite3` */ + databaseUrl?: string; +} /** * Prevent messages with the exact same content from being submitted repeatedly. * It stores a hashcode for each content in an SQLite database and rate-limits them. * Only messages that meet the minimum length criteria are selected. */ -const antiDuplicationPolicy: Policy = async (msg) => { +const antiDuplicationPolicy: Policy = async (msg, opts) => { + const ttl = opts?.ttl ?? 60000; + const minLength = opts?.minLength ?? 50; + const databaseUrl = opts?.databaseUrl || 'sqlite:///tmp/strfry-anti-duplication-policy.sqlite3'; + const { kind, content } = msg.event; - if (kind === 1 && content.length >= ANTI_DUPLICATION_MIN_LENGTH) { - const db = new Keydb('sqlite:///tmp/strfry-anti-duplication-policy.sqlite3'); + if (kind === 1 && content.length >= minLength) { + const db = new Keydb(databaseUrl); const hash = String(hashCode(content)); if (await db.get(hash)) { - await db.set(hash, 1, ANTI_DUPLICATION_TTL); + await db.set(hash, 1, ttl); return { id: msg.event.id, action: 'shadowReject', @@ -26,7 +36,7 @@ const antiDuplicationPolicy: Policy = async (msg) => { }; } - await db.set(hash, 1, ANTI_DUPLICATION_TTL); + await db.set(hash, 1, ttl); } return { @@ -52,3 +62,5 @@ function hashCode(str: string): number { } export default antiDuplicationPolicy; + +export type { AntiDuplication }; diff --git a/src/policies/hellthread-policy.ts b/src/policies/hellthread-policy.ts index bb980ee..9f05171 100755 --- a/src/policies/hellthread-policy.ts +++ b/src/policies/hellthread-policy.ts @@ -1,12 +1,13 @@ import type { Policy } from '../types.ts'; interface Hellthread { - limit: number; + /** Total number of "p" tags a kind 1 note may have before it's rejected. Default: `100` */ + limit?: number; } /** Reject messages that tag too many participants. */ const hellthreadPolicy: Policy = (msg, opts) => { - const limit = opts?.limit || 100; + const limit = opts?.limit ?? 100; if (msg.event.kind === 1) { const p = msg.event.tags.filter((tag) => tag[0] === 'p'); @@ -28,3 +29,5 @@ const hellthreadPolicy: Policy = (msg, opts) => { }; export default hellthreadPolicy; + +export type { Hellthread }; diff --git a/src/policies/noop-policy.ts b/src/policies/noop-policy.ts index a930ef9..0d5e86d 100755 --- a/src/policies/noop-policy.ts +++ b/src/policies/noop-policy.ts @@ -4,7 +4,7 @@ import type { Policy } from '../types.ts'; * Minimal sample policy for demonstration purposes. * Allows all events through. */ -const noopPolicy: Policy = (msg) => ({ +const noopPolicy: Policy = (msg) => ({ id: msg.event.id, action: 'accept', msg: '', diff --git a/src/policies/rate-limit-policy.ts b/src/policies/rate-limit-policy.ts index a6810bb..f368bd6 100755 --- a/src/policies/rate-limit-policy.ts +++ b/src/policies/rate-limit-policy.ts @@ -2,23 +2,34 @@ import { Keydb } from '../deps.ts'; import type { Policy } from '../types.ts'; -const IP_WHITELIST = (Deno.env.get('IP_WHITELIST') || '').split(','); - -const RATE_LIMIT_INTERVAL = Number(Deno.env.get('RATE_LIMIT_INTERVAL') || 60000); -const RATE_LIMIT_MAX = Number(Deno.env.get('RATE_LIMIT_MAX') || 10); +interface RateLimit { + /** How often (ms) to check whether `max` has been exceeded. Default: `60000` (1 minute). */ + interval?: number; + /** Max number of requests within the `interval` until the IP is rate-limited. Default: `10`. */ + max?: number; + /** List of IP addresses to skip this policy. */ + whitelist?: string[]; + /** Database connection string. Default: `sqlite:///tmp/strfry-rate-limit-policy.sqlite3` */ + databaseUrl?: string; +} /** * Rate-limits users by their IP address. * IPs are stored in an SQLite database. If you are running internal services, * it's a good idea to at least whitelist `127.0.0.1` etc. */ -const rateLimitPolicy: Policy = async (msg) => { - if ((msg.sourceType === 'IP4' || msg.sourceType === 'IP6') && !IP_WHITELIST.includes(msg.sourceInfo)) { - const db = new Keydb('sqlite:///tmp/strfry-rate-limit-policy.sqlite3'); - const count = await db.get(msg.sourceInfo) || 0; - await db.set(msg.sourceInfo, count + 1, RATE_LIMIT_INTERVAL); +const rateLimitPolicy: Policy = async (msg, opts) => { + const interval = opts?.interval ?? 60000; + const max = opts?.max ?? 10; + const whitelist = opts?.whitelist || []; + const databaseUrl = opts?.databaseUrl || 'sqlite:///tmp/strfry-rate-limit-policy.sqlite3'; - if (count >= RATE_LIMIT_MAX) { + if ((msg.sourceType === 'IP4' || msg.sourceType === 'IP6') && !whitelist.includes(msg.sourceInfo)) { + const db = new Keydb(databaseUrl); + const count = await db.get(msg.sourceInfo) || 0; + await db.set(msg.sourceInfo, count + 1, interval); + + if (count >= max) { return { id: msg.event.id, action: 'reject', @@ -35,3 +46,5 @@ const rateLimitPolicy: Policy = async (msg) => { }; export default rateLimitPolicy; + +export type { RateLimit }; diff --git a/src/policies/read-only-policy.ts b/src/policies/read-only-policy.ts index 996c27b..34fb709 100755 --- a/src/policies/read-only-policy.ts +++ b/src/policies/read-only-policy.ts @@ -1,7 +1,7 @@ import type { Policy } from '../types.ts'; /** This policy rejects all messages. */ -const readOnlyPolicy: Policy = (msg) => ({ +const readOnlyPolicy: Policy = (msg) => ({ id: msg.event.id, action: 'reject', msg: 'The relay is read-only.', From 84ad093409e50baec2caa88d4dfe631b840b343c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 20:51:30 -0500 Subject: [PATCH 13/17] Improve example entrypoint --- entrypoint.example.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.example.ts b/entrypoint.example.ts index 8ec5df6..875dd85 100644 --- a/entrypoint.example.ts +++ b/entrypoint.example.ts @@ -15,8 +15,8 @@ const msg = await readStdin(); const result = await pipeline(msg, [ noopPolicy, [hellthreadPolicy, { limit: 100 }], - antiDuplicationPolicy, - rateLimitPolicy, + [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }], + [rateLimitPolicy, { whitelist: ['127.0.0.1'] }], ]); writeStdout(result); From 2330349ad62ebca39a64d961036a557b7f702e5b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 24 Mar 2023 22:33:12 -0500 Subject: [PATCH 14/17] Fix types... Jesus --- src/pipeline.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index ff0469e..650eafe 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,7 +1,9 @@ import { InputMessage, OutputMessage, Policy } from './types.ts'; /** A policy function with opts to run it with. Used by the pipeline. */ -type PolicyTuple = [policy: Policy, opts?: Opts]; +type PolicyTuple

= [policy: P, opts?: InferPolicyOpts

]; +/** Infer opts from the policy. */ +type InferPolicyOpts

= P extends Policy ? Opts : never; /** Helper type for proper type inference of PolicyTuples in the pipeline. */ // https://stackoverflow.com/a/75806165 @@ -28,7 +30,7 @@ async function pipeline(msg: InputMessage, policies: [...Policy } /** Coerce item into a tuple if it isn't already. */ -function toTuple(item: PolicyTuple | Policy): PolicyTuple { +function toTuple

(item: PolicyTuple

| P): PolicyTuple

{ return typeof item === 'function' ? [item] : item; } From c6c7b759e67727e8c358f0fbfd49b5928b3811cc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 25 Mar 2023 08:24:47 -0500 Subject: [PATCH 15/17] Type tweaks --- src/pipeline.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 650eafe..dc0e62d 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,16 +5,16 @@ type PolicyTuple

= [policy: P, opts?: InferPolicyOpts /** Infer opts from the policy. */ type InferPolicyOpts

= P extends Policy ? Opts : never; -/** Helper type for proper type inference of PolicyTuples in the pipeline. */ +/** Helper type for proper type inference of PolicyTuples. */ // https://stackoverflow.com/a/75806165 // https://stackoverflow.com/a/54608401 -type PolicySpread = { +type Policies = { [K in keyof T]: PolicyTuple | Policy; }; /** Processes messages through multiple policies, bailing early on rejection. */ -async function pipeline(msg: InputMessage, policies: [...PolicySpread]): Promise { - for (const item of policies) { +async function pipeline(msg: InputMessage, policies: [...Policies]): Promise { + for (const item of policies as (Policy | PolicyTuple)[]) { const [policy, opts] = toTuple(item); const result = await policy(msg, opts); if (result.action !== 'accept') { From 37ce2db08956bf4641f48dbb2f9a8a51fed60730 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 25 Mar 2023 10:06:40 -0500 Subject: [PATCH 16/17] Add a lot more detail to the README --- README.md | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b993bac..32f614e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,183 @@ # strfry policies -A collection of policies for the [strfry](https://github.com/hoytech/strfry) relay, written in Deno. +A collection of policies for the [strfry](https://github.com/hoytech/strfry) Nostr relay, built in Deno. For more information about installing these policies and how they work, see [Write policy plugins](https://github.com/hoytech/strfry/blob/master/docs/plugins.md). +This library introduces a model for writing policies and composing them in a pipeline. Policies are fully configurable and it's easy to add your own or install more from anywhere on the net. + +## Getting started + +To get up and running, you will need to install Deno on the same machine as strfry: + +```sh +sudo apt install -y unzip +curl -fsSL https://deno.land/x/install/install.sh | sudo DENO_INSTALL=/usr/local sh +``` + +Create an entrypoint file somewhere and make it executable: + +```sh +sudo touch /opt/strfry-policy.ts +sudo chmod +x /opt/strfry-policy.ts +``` + +Now you can write your policy. Here's a good starting point: + +```ts +#!/bin/sh +//bin/true; exec deno run -A "$0" "$@" +import { + antiDuplicationPolicy, + hellthreadPolicy, + pipeline, + rateLimitPolicy, + readStdin, + writeStdout, +} from 'https://gitlab.com/soapbox-pub/strfry-policies/-/blob/develop/mod.ts'; + +const msg = await readStdin(); + +const result = await pipeline(msg, [ + [hellthreadPolicy, { limit: 100 }], + [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }], + [rateLimitPolicy, { whitelist: ['127.0.0.1'] }], +]); + +writeStdout(result); +``` + +Finally, edit `strfry.conf` and enable the policy: + +```diff + writePolicy { + # If non-empty, path to an executable script that implements the writePolicy plugin logic +- plugin = "" ++ plugin = "/opt/strfry-policy.ts" + + # Number of seconds to search backwards for lookback events when starting the writePolicy plugin (0 for no lookback) + lookbackSeconds = 0 +``` + +That's it! 🎉 Now you should check strfry logs to ensure everything is working okay. + +## Writing your own policies + +You can write a policy in TypeScript and host it anywhere. Deno allows importing modules by URL, making it easy to share policies. + +Here is a basic sample policy: + +```ts +import type { Policy } from 'https://gitlab.com/soapbox-pub/strfry-policies/-/blob/develop/mod.ts'; + +/** Only American English is allowed. */ +const americanPolicy: Policy = (msg) => { + const { content } = msg.event; + + const words = [ + 'armour', + 'behaviour', + 'colour', + 'favourite', + 'flavour', + 'honour', + 'humour', + 'rumour', + ]; + + const isBritish = words.some((word) => content.toLowerCase().includes(word)); + + if (isBritish) { + return { + id: msg.event.id, + action: 'reject', + msg: 'Sorry, only American English is allowed on this server!', + }; + } else { + return { + id: msg.event.id, + action: 'accept', + msg: '', + }; + } +}; + +export default americanPolicy; +``` + +Once you're done, you can either upload the file somewhere online or directly to your server. Then, update your pipeline: + +```diff +--- a/strfry-policy.ts ++++ b/strfry-policy.ts +@@ -9,6 +9,7 @@ import { + readStdin, + writeStdout, + } from 'https://gitlab.com/soapbox-pub/strfry-policies/-/blob/develop/mod.ts'; ++import { americanPolicy } from 'https://gist.githubusercontent.com/alexgleason/5c2d084434fa0875397f44da198f4352/raw/3d3ce71c7ed9cef726f17c3a102c378b81760a45/american-policy.ts'; + + const msg = await readStdin(); + +@@ -17,6 +18,7 @@ const result = await pipeline(msg, [ + [hellthreadPolicy, { limit: 100 }], + [antiDuplicationPolicy, { ttl: 60000, minLength: 50 }], + [rateLimitPolicy, { whitelist: ['127.0.0.1'] }], ++ americanPolicy, + ]); + + writeStdout(result); +``` + +### Policy options + +The `Policy` type is a generic that accepts options of any type. With opts, the policy above could be rewritten as: + +```diff +--- a/american-policy.ts ++++ b/american-policy.ts +@@ -1,7 +1,11 @@ + import type { Policy } from 'https://gitlab.com/soapbox-pub/strfry-policies/-/blob/develop/mod.ts'; + ++interface American { ++ withGrey?: boolean; ++} ++ + /** Only American English is allowed. */ +-const americanPolicy: Policy = (msg) => { ++const americanPolicy: Policy = (msg, opts) => { + const { content } = msg.event; + + const words = [ +@@ -15,6 +19,10 @@ + 'rumour', + ]; + ++ if (opts?.withGrey) { ++ words.push('grey'); ++ } ++ + const isBritish = words.some((word) => content.toLowerCase().includes(word)); + + if (isBritish) { +``` + +Then, in the pipeline: + +```diff +- americanPolicy, ++ [americanPolicy, { withGrey: true }], +``` + +### Caveats + +- You should not use `console.log` anywhere in your policies, as strfry expects stdout to be the strfry output message. + +## Available policies + +Please look directly at `src/policies` in this repo. The files include detailed JSDoc comments and it has good type support. + +![Policies TypeScript](https://gitlab.com/soapbox-pub/strfry-policies/uploads/dfb993b3464af5ed78bb8e5db8677458/Kazam_screencast_00090.webm) + ## License This is free and unencumbered software released into the public domain. From 8030fe641ad2245c371d79f8a7f92b996e1cc21f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 25 Mar 2023 10:15:06 -0500 Subject: [PATCH 17/17] Add GitLab CI and a simple test --- .gitlab-ci.yml | 19 +++++++++ deno.json | 1 - deno.lock | 58 +++++++++++++++++++++++++++ src/deps.ts | 3 +- src/policies/read-only-policy.test.ts | 23 +++++++++++ 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 deno.lock create mode 100644 src/policies/read-only-policy.test.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..9464840 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,19 @@ +image: denoland/deno:1.32.1 + +default: + interruptible: true + +stages: + - test + +fmt: + stage: test + script: deno fmt --check + +lint: + stage: test + script: deno lint + +test: + stage: test + script: deno test \ No newline at end of file diff --git a/deno.json b/deno.json index 0b7b419..b5508ad 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,4 @@ { - "lock": false, "tasks": { }, "lint": { diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..0a80d27 --- /dev/null +++ b/deno.lock @@ -0,0 +1,58 @@ +{ + "version": "2", + "remote": { + "https://deno.land/std@0.181.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", + "https://deno.land/std@0.181.0/bytes/bytes_list.ts": "b4cbdfd2c263a13e8a904b12d082f6177ea97d9297274a4be134e989450dfa6a", + "https://deno.land/std@0.181.0/bytes/concat.ts": "d26d6f3d7922e6d663dacfcd357563b7bf4a380ce5b9c2bbe0c8586662f25ce2", + "https://deno.land/std@0.181.0/bytes/copy.ts": "939d89e302a9761dcf1d9c937c7711174ed74c59eef40a1e4569a05c9de88219", + "https://deno.land/std@0.181.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e", + "https://deno.land/std@0.181.0/io/buf_reader.ts": "abeb92b18426f11d72b112518293a96aef2e6e55f80b84235e8971ac910affb5", + "https://deno.land/std@0.181.0/io/buf_writer.ts": "48c33c8f00b61dcbc7958706741cec8e59810bd307bc6a326cbd474fe8346dfd", + "https://deno.land/std@0.181.0/io/buffer.ts": "17f4410eaaa60a8a85733e8891349a619eadfbbe42e2f319283ce2b8f29723ab", + "https://deno.land/std@0.181.0/io/copy_n.ts": "0cc7ce07c75130f6fc18621ec1911c36e147eb9570664fee0ea12b1988167590", + "https://deno.land/std@0.181.0/io/limited_reader.ts": "6c9a216f8eef39c1ee2a6b37a29372c8fc63455b2eeb91f06d9646f8f759fc8b", + "https://deno.land/std@0.181.0/io/mod.ts": "2665bcccc1fd6e8627cca167c3e92aaecbd9897556b6f69e6d258070ef63fd9b", + "https://deno.land/std@0.181.0/io/multi_reader.ts": "9c2a0a31686c44b277e16da1d97b4686a986edcee48409b84be25eedbc39b271", + "https://deno.land/std@0.181.0/io/read_delim.ts": "c02b93cc546ae8caad8682ae270863e7ace6daec24c1eddd6faabc95a9d876a3", + "https://deno.land/std@0.181.0/io/read_int.ts": "7cb8bcdfaf1107586c3bacc583d11c64c060196cb070bb13ae8c2061404f911f", + "https://deno.land/std@0.181.0/io/read_lines.ts": "c526c12a20a9386dc910d500f9cdea43cba974e853397790bd146817a7eef8cc", + "https://deno.land/std@0.181.0/io/read_long.ts": "f0aaa420e3da1261c5d33c5e729f09922f3d9fa49f046258d4ff7a00d800c71e", + "https://deno.land/std@0.181.0/io/read_range.ts": "28152daf32e43dd9f7d41d8466852b0d18ad766cd5c4334c91fef6e1b3a74eb5", + "https://deno.land/std@0.181.0/io/read_short.ts": "805cb329574b850b84bf14a92c052c59b5977a492cd780c41df8ad40826c1a20", + "https://deno.land/std@0.181.0/io/read_string_delim.ts": "5dc9f53bdf78e7d4ee1e56b9b60352238ab236a71c3e3b2a713c3d78472a53ce", + "https://deno.land/std@0.181.0/io/slice_long_to_bytes.ts": "48d9bace92684e880e46aa4a2520fc3867f9d7ce212055f76ecc11b22f9644b7", + "https://deno.land/std@0.181.0/io/string_reader.ts": "da0f68251b3d5b5112485dfd4d1b1936135c9b4d921182a7edaf47f74c25cc8f", + "https://deno.land/std@0.181.0/io/string_writer.ts": "8a03c5858c24965a54c6538bed15f32a7c72f5704a12bda56f83a40e28e5433e", + "https://deno.land/std@0.181.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", + "https://deno.land/std@0.181.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", + "https://deno.land/std@0.181.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f", + "https://deno.land/std@0.181.0/types.d.ts": "dbaeb2c4d7c526db9828fc8df89d8aecf53b9ced72e0c4568f97ddd8cda616a4", + "https://deno.land/std@0.86.0/async/deferred.ts": "f89ed49ba5e1dd0227c6bd5b23f017be46c3f92e4f0338dda08ff5aa54b9f6c9", + "https://deno.land/std@0.86.0/async/delay.ts": "9de1d8d07d1927767ab7f82434b883f3d8294fb19cad819691a2ad81a728cf3d", + "https://deno.land/std@0.86.0/async/mod.ts": "253b41c658d768613eacfb11caa0a9ca7148442f932018a45576f7f27554c853", + "https://deno.land/std@0.86.0/async/mux_async_iterator.ts": "b9091909db04cdb0af6f7807677372f64c1488de6c4bd86004511b064bf230d6", + "https://deno.land/std@0.86.0/async/pool.ts": "876f9e6815366cd017a3b4fbb9e9ae40310b1b6972f1bd541c94358bc11fb7e5", + "https://deno.land/std@0.86.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e", + "https://deno.land/std@0.86.0/encoding/hex.ts": "f952e0727bddb3b2fd2e6889d104eacbd62e92091f540ebd6459317a61932d9b", + "https://deno.land/std@0.86.0/fmt/colors.ts": "d253f2367e5feebcf0f49676533949c09ea07a2d95fb74958f6f5c1c22fbec49", + "https://deno.land/std@0.86.0/node/_utils.ts": "067c386d676432e9418808851e8de72df7774f009a652904f62358b4c94504cf", + "https://deno.land/std@0.86.0/node/buffer.ts": "e98af24a3210d8fc3f022b6eb26d6e5bdf98fb0e02931e5983d20db9fed1b590", + "https://deno.land/std@0.86.0/testing/_diff.ts": "961eaf6d9f5b0a8556c9d835bbc6fa74f5addd7d3b02728ba7936ff93364f7a3", + "https://deno.land/std@0.86.0/testing/asserts.ts": "de942b2e1cb6dac1c1c4a7b698be017337e2e2cc2252ae0a6d215c5befde1e82", + "https://deno.land/x/keydb@1.0.0/adapter.ts": "7492c993a6bd5ea033a3423e6383faa34bafd78292cfa344955e263d650112bc", + "https://deno.land/x/keydb@1.0.0/jsonb.ts": "05fa1b45f43ea5e755a2271ef3dbeb3e4c64da7053c0c6c53e9d994fa98d9b72", + "https://deno.land/x/keydb@1.0.0/keydb.ts": "616c4c866c9e11c29d5654d367468ed51b689565043f53fdeb5eb66f25138156", + "https://deno.land/x/keydb@1.0.0/memory.ts": "f0ab6faf293c4ad3539fd3cf89c764d7f34d39d24e471ea59eebb5d1f5a510dc", + "https://deno.land/x/keydb@1.0.0/sqlite.ts": "a16e0242077a0bd1bf027f5e5440778c237d5ff2e278aeebb2245c407a4e3b43", + "https://deno.land/x/module_cache@0.0.1/mod.ts": "542ea337ec9c5ef4013a2c49a42d9faa9d161898c33629594ee526d883bc2cc6", + "https://deno.land/x/sqlite@v2.3.2/build/sqlite.js": "24d7663da39aff93cb7a1c9fef28346d7c457ba30bba7fccfb211a94b4823c29", + "https://deno.land/x/sqlite@v2.3.2/build/vfs.js": "ce3b69b82a1723cad2bf06dfee774f461bcc49502a20ec8b99198ceff46cd046", + "https://deno.land/x/sqlite@v2.3.2/mod.ts": "1fc2fdc6aca3ffa8b3f4893c2858871e1bb24ab833d1ed732eba911ad44fb495", + "https://deno.land/x/sqlite@v2.3.2/src/constants.ts": "6032b1a8b2e6ed186460385b83750deb8e9b7b8c6ff100e450605c814585186e", + "https://deno.land/x/sqlite@v2.3.2/src/db.ts": "f5269a3014907b6758f99c6ac08b6dc1b448415703e745fb57ea3d3f8adb2a97", + "https://deno.land/x/sqlite@v2.3.2/src/error.ts": "c305a89b28ab5b56e425074849c6544d23e855b92670935d9e965ee68f9a6a9e", + "https://deno.land/x/sqlite@v2.3.2/src/row_objects.ts": "cf7ad165bb14c0fd346c46a7f3ea08043ace747d1abae3787bbe4b36be11b09c", + "https://deno.land/x/sqlite@v2.3.2/src/rows.ts": "1b05730096f8df626bfc9d1597a776cc5ac7d989910a20711fa28c7a8daebb0e", + "https://deno.land/x/sqlite@v2.3.2/src/wasm.ts": "9747b8c4de5542f2359a35461317f08a244fe8a0933c7e279caad27a0190e6bf" + } +} diff --git a/src/deps.ts b/src/deps.ts index 0544dde..9403356 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,2 +1,3 @@ -export { readLines } from 'https://deno.land/std@0.178.0/io/mod.ts'; +export { readLines } from 'https://deno.land/std@0.181.0/io/mod.ts'; +export { assert } from 'https://deno.land/std@0.181.0/testing/asserts.ts'; export { Keydb } from 'https://deno.land/x/keydb@1.0.0/sqlite.ts'; diff --git a/src/policies/read-only-policy.test.ts b/src/policies/read-only-policy.test.ts new file mode 100644 index 0000000..ee61c75 --- /dev/null +++ b/src/policies/read-only-policy.test.ts @@ -0,0 +1,23 @@ +import { assert } from '../deps.ts'; + +import readOnlyPolicy from './read-only-policy.ts'; + +Deno.test('always rejects', async () => { + const result = await readOnlyPolicy({ + event: { + kind: 1, + id: '', + content: '', + created_at: 0, + pubkey: '', + sig: '', + tags: [], + }, + receivedAt: 0, + sourceInfo: '127.0.0.1', + sourceType: 'IP4', + type: 'new', + }); + + assert(result.action === 'reject'); +});