Add OpenAI policy

deno.lock

@@ -155,6 +155,13 @@
     "https://deno.land/std@0.86.0/node/buffer.ts": "e98af24a3210d8fc3f022b6eb26d6e5bdf98fb0e02931e5983d20db9fed1b590",
     "https://deno.land/std@0.86.0/testing/_diff.ts": "961eaf6d9f5b0a8556c9d835bbc6fa74f5addd7d3b02728ba7936ff93364f7a3",
     "https://deno.land/std@0.86.0/testing/asserts.ts": "de942b2e1cb6dac1c1c4a7b698be017337e2e2cc2252ae0a6d215c5befde1e82",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.error.ts": "c75e83b959d0b4b18bc5434c15dc5ab40a37fe6069cb51d7607a22391a7b22af",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.ts": "bc06fb96910aea4c042b7c7387887fecd70f272b7230de0262711c878066111c",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.type.ts": "c03dc0cb93b4e68496813e9c0a322e3e2a08d16fb78f9b7196a5328cbc0ed771",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-interceptor.ts": "659014b78275a679fc4644d1f36ba486a27eaa7149c2f3f32fa34a2e7a085059",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-scope.ts": "f7235b1efa7371b0698f0b5122eb82d042063411429585e4cdf740f2ae60af94",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mock-utils.ts": "3a7b12704a75eee2e9721221c3da80bd70ab4f00fdbd1b786fb240f1be20b497",
+    "https://deno.land/x/deno_mock_fetch@1.0.1/mod.ts": "7ea9e35228454b578ff47d42602eba53de0174ffa4ca5567308b11384b55ddd7",
     "https://deno.land/x/keydb@1.0.0/adapter.ts": "7492c993a6bd5ea033a3423e6383faa34bafd78292cfa344955e263d650112bc",
     "https://deno.land/x/keydb@1.0.0/jsonb.ts": "05fa1b45f43ea5e755a2271ef3dbeb3e4c64da7053c0c6c53e9d994fa98d9b72",
     "https://deno.land/x/keydb@1.0.0/keydb.ts": "616c4c866c9e11c29d5654d367468ed51b689565043f53fdeb5eb66f25138156",

mod.ts

@@ -3,6 +3,7 @@ export { default as filterPolicy, type Filter } from './src/policies/filter-poli
 export { default as hellthreadPolicy, type Hellthread } from './src/policies/hellthread-policy.ts';
 export { default as keywordPolicy } from './src/policies/keyword-policy.ts';
 export { default as noopPolicy } from './src/policies/noop-policy.ts';
+export { default as openaiPolicy, type OpenAI, type OpenAIHandler } from './src/policies/openai-policy.ts';
 export { default as pubkeyBanPolicy } from './src/policies/pubkey-ban-policy.ts';
 export { default as rateLimitPolicy, type RateLimit } from './src/policies/rate-limit-policy.ts';
 export { default as readOnlyPolicy } from './src/policies/read-only-policy.ts';

src/policies/openai-policy.test.ts

@@ -0,0 +1,32 @@
+import { MockFetch } from 'https://deno.land/x/deno_mock_fetch@1.0.1/mod.ts';
+
+import { assertEquals } from '../deps.ts';
+import { buildEvent, buildInputMessage } from '../test.ts';
+
+import openaiPolicy from './openai-policy.ts';
+
+const mockFetch = new MockFetch();
+
+mockFetch.deactivateNetConnect();
+
+mockFetch
+  .intercept('https://api.openai.com/v1/moderations', { body: '{"input":"I want to kill them."}' })
+  .response(
+    '{"id":"modr-6zvK0JiWLBpJvA5IrJufw8BHPpEpB","model":"text-moderation-004","results":[{"flagged":true,"categories":{"sexual":false,"hate":false,"violence":true,"self-harm":false,"sexual/minors":false,"hate/threatening":false,"violence/graphic":false},"category_scores":{"sexual":9.759669410414062e-07,"hate":0.180674210190773,"violence":0.8864424824714661,"self-harm":1.8088556208439854e-09,"sexual/minors":1.3363569806301712e-08,"hate/threatening":0.003288434585556388,"violence/graphic":3.2010063932830235e-08}}]}',
+  );
+
+mockFetch
+  .intercept('https://api.openai.com/v1/moderations', { body: '{"input":"I want to love them."}' })
+  .response(
+    '{"id":"modr-6zvS6HoiwBqOQ9VYSggGAAI9vSgWD","model":"text-moderation-004","results":[{"flagged":false,"categories":{"sexual":false,"hate":false,"violence":false,"self-harm":false,"sexual/minors":false,"hate/threatening":false,"violence/graphic":false},"category_scores":{"sexual":1.94798508346139e-06,"hate":2.756592039077077e-07,"violence":1.4010063864589029e-07,"self-harm":3.1806530742528594e-09,"sexual/minors":1.8928545841845335e-08,"hate/threatening":3.1036221769670247e-12,"violence/graphic":1.5348690096672613e-09}}]}',
+  );
+
+Deno.test('rejects flagged events', async () => {
+  const msg = buildInputMessage({ event: buildEvent({ content: 'I want to kill them.' }) });
+  assertEquals((await openaiPolicy(msg)).action, 'reject');
+});
+
+Deno.test('accepts unflagged events', async () => {
+  const msg = buildInputMessage({ event: buildEvent({ content: 'I want to love them.' }) });
+  assertEquals((await openaiPolicy(msg)).action, 'accept');
+});

src/policies/openai-policy.ts

@@ -0,0 +1,115 @@
+import type { Event, Policy } from '../types.ts';
+
+/**
+ * OpenAI moderation result.
+ *
+ * https://platform.openai.com/docs/api-reference/moderations/create
+ */
+interface ModerationData {
+  id: string;
+  model: string;
+  results: {
+    categories: {
+      'hate': boolean;
+      'hate/threatening': boolean;
+      'self-harm': boolean;
+      'sexual': boolean;
+      'sexual/minors': boolean;
+      'violence': boolean;
+      'violence/graphic': boolean;
+    };
+    category_scores: {
+      'hate': number;
+      'hate/threatening': number;
+      'self-harm': number;
+      'sexual': number;
+      'sexual/minors': number;
+      'violence': number;
+      'violence/graphic': number;
+    };
+    flagged: boolean;
+  }[];
+}
+
+/**
+ * Callback for fine control over the policy. It contains the event and the OpenAI moderation data.
+ * Implementations should return `true` to **reject** the content, and `false` to accept.
+ */
+type OpenAIHandler = (event: Event, data: ModerationData) => boolean;
+
+/** Policy options for `openaiPolicy`. */
+interface OpenAI {
+  handler?: OpenAIHandler;
+  endpoint?: string;
+  apiKey?: string;
+}
+
+/** Default handler. Simply checks whether OpenAI flagged the content. */
+const flaggedHandler: OpenAIHandler = (_, { results }) => results.some((r) => r.flagged);
+
+/**
+ * Passes event content to OpenAI and then rejects flagged events.
+ *
+ * By default, this policy will reject kind 1 events that OpenAI flags.
+ * It's possible to pass a custom handler for more control. An OpenAI API key is required.
+ *
+ * @example
+ * ```ts
+ * // Default handler. It's so strict it's suitable for school children.
+ * openaiPolicy(msg, { apiKey: Deno.env.get('OPENAI_API_KEY') });
+ *
+ * // With a custom handler.
+ * openaiPolicy(msg, {
+ *   apiKey: Deno.env.get('OPENAI_API_KEY'),
+ *   handler(event, result) {
+ *     // Loop each result.
+ *     return data.results.some((result) => {
+ *       if (result.flagged) {
+ *         const { sexual, violence } = result.categories;
+ *         // Reject only events flagged as sexual and violent.
+ *         return sexual && violence;
+ *       }
+ *     });
+ *   },
+ * });
+ * ```
+ */
+const openaiPolicy: Policy<OpenAI> = async ({ event }, opts = {}) => {
+  const {
+    handler = flaggedHandler,
+    endpoint = 'https://api.openai.com/v1/moderations',
+    apiKey,
+  } = opts;
+
+  if (event.kind === 1) {
+    const resp = await fetch(endpoint, {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        input: event.content,
+      }),
+    });
+
+    const result = await resp.json();
+
+    if (handler(event, result)) {
+      return {
+        id: event.id,
+        action: 'reject',
+        msg: 'blocked: content flagged by AI',
+      };
+    }
+  }
+
+  return {
+    id: event.id,
+    action: 'accept',
+    msg: '',
+  };
+};
+
+export { flaggedHandler, openaiPolicy as default };
+
+export type { ModerationData, OpenAI, OpenAIHandler };