diff --git a/deno.lock b/deno.lock index af01131..a6e2211 100644 --- a/deno.lock +++ b/deno.lock @@ -155,6 +155,13 @@ "https://deno.land/std@0.86.0/node/buffer.ts": "e98af24a3210d8fc3f022b6eb26d6e5bdf98fb0e02931e5983d20db9fed1b590", "https://deno.land/std@0.86.0/testing/_diff.ts": "961eaf6d9f5b0a8556c9d835bbc6fa74f5addd7d3b02728ba7936ff93364f7a3", "https://deno.land/std@0.86.0/testing/asserts.ts": "de942b2e1cb6dac1c1c4a7b698be017337e2e2cc2252ae0a6d215c5befde1e82", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.error.ts": "c75e83b959d0b4b18bc5434c15dc5ab40a37fe6069cb51d7607a22391a7b22af", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.ts": "bc06fb96910aea4c042b7c7387887fecd70f272b7230de0262711c878066111c", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-fetch.type.ts": "c03dc0cb93b4e68496813e9c0a322e3e2a08d16fb78f9b7196a5328cbc0ed771", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-interceptor.ts": "659014b78275a679fc4644d1f36ba486a27eaa7149c2f3f32fa34a2e7a085059", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-scope.ts": "f7235b1efa7371b0698f0b5122eb82d042063411429585e4cdf740f2ae60af94", + "https://deno.land/x/deno_mock_fetch@1.0.1/mock-utils.ts": "3a7b12704a75eee2e9721221c3da80bd70ab4f00fdbd1b786fb240f1be20b497", + "https://deno.land/x/deno_mock_fetch@1.0.1/mod.ts": "7ea9e35228454b578ff47d42602eba53de0174ffa4ca5567308b11384b55ddd7", "https://deno.land/x/keydb@1.0.0/adapter.ts": "7492c993a6bd5ea033a3423e6383faa34bafd78292cfa344955e263d650112bc", "https://deno.land/x/keydb@1.0.0/jsonb.ts": "05fa1b45f43ea5e755a2271ef3dbeb3e4c64da7053c0c6c53e9d994fa98d9b72", "https://deno.land/x/keydb@1.0.0/keydb.ts": "616c4c866c9e11c29d5654d367468ed51b689565043f53fdeb5eb66f25138156", diff --git a/mod.ts b/mod.ts index 628a6d2..e3addcb 100644 --- a/mod.ts +++ b/mod.ts @@ -3,6 +3,7 @@ export { default as filterPolicy, type Filter } from './src/policies/filter-poli export { default as hellthreadPolicy, type Hellthread } from './src/policies/hellthread-policy.ts'; export { default as keywordPolicy } from './src/policies/keyword-policy.ts'; export { default as noopPolicy } from './src/policies/noop-policy.ts'; +export { default as openaiPolicy, type OpenAI, type OpenAIHandler } from './src/policies/openai-policy.ts'; export { default as pubkeyBanPolicy } from './src/policies/pubkey-ban-policy.ts'; export { default as rateLimitPolicy, type RateLimit } from './src/policies/rate-limit-policy.ts'; export { default as readOnlyPolicy } from './src/policies/read-only-policy.ts'; diff --git a/src/policies/openai-policy.test.ts b/src/policies/openai-policy.test.ts new file mode 100644 index 0000000..cb5cb22 --- /dev/null +++ b/src/policies/openai-policy.test.ts @@ -0,0 +1,32 @@ +import { MockFetch } from 'https://deno.land/x/deno_mock_fetch@1.0.1/mod.ts'; + +import { assertEquals } from '../deps.ts'; +import { buildEvent, buildInputMessage } from '../test.ts'; + +import openaiPolicy from './openai-policy.ts'; + +const mockFetch = new MockFetch(); + +mockFetch.deactivateNetConnect(); + +mockFetch + .intercept('https://api.openai.com/v1/moderations', { body: '{"input":"I want to kill them."}' }) + .response( + '{"id":"modr-6zvK0JiWLBpJvA5IrJufw8BHPpEpB","model":"text-moderation-004","results":[{"flagged":true,"categories":{"sexual":false,"hate":false,"violence":true,"self-harm":false,"sexual/minors":false,"hate/threatening":false,"violence/graphic":false},"category_scores":{"sexual":9.759669410414062e-07,"hate":0.180674210190773,"violence":0.8864424824714661,"self-harm":1.8088556208439854e-09,"sexual/minors":1.3363569806301712e-08,"hate/threatening":0.003288434585556388,"violence/graphic":3.2010063932830235e-08}}]}', + ); + +mockFetch + .intercept('https://api.openai.com/v1/moderations', { body: '{"input":"I want to love them."}' }) + .response( + '{"id":"modr-6zvS6HoiwBqOQ9VYSggGAAI9vSgWD","model":"text-moderation-004","results":[{"flagged":false,"categories":{"sexual":false,"hate":false,"violence":false,"self-harm":false,"sexual/minors":false,"hate/threatening":false,"violence/graphic":false},"category_scores":{"sexual":1.94798508346139e-06,"hate":2.756592039077077e-07,"violence":1.4010063864589029e-07,"self-harm":3.1806530742528594e-09,"sexual/minors":1.8928545841845335e-08,"hate/threatening":3.1036221769670247e-12,"violence/graphic":1.5348690096672613e-09}}]}', + ); + +Deno.test('rejects flagged events', async () => { + const msg = buildInputMessage({ event: buildEvent({ content: 'I want to kill them.' }) }); + assertEquals((await openaiPolicy(msg)).action, 'reject'); +}); + +Deno.test('accepts unflagged events', async () => { + const msg = buildInputMessage({ event: buildEvent({ content: 'I want to love them.' }) }); + assertEquals((await openaiPolicy(msg)).action, 'accept'); +}); diff --git a/src/policies/openai-policy.ts b/src/policies/openai-policy.ts new file mode 100755 index 0000000..4d5b847 --- /dev/null +++ b/src/policies/openai-policy.ts @@ -0,0 +1,115 @@ +import type { Event, Policy } from '../types.ts'; + +/** + * OpenAI moderation result. + * + * https://platform.openai.com/docs/api-reference/moderations/create + */ +interface ModerationData { + id: string; + model: string; + results: { + categories: { + 'hate': boolean; + 'hate/threatening': boolean; + 'self-harm': boolean; + 'sexual': boolean; + 'sexual/minors': boolean; + 'violence': boolean; + 'violence/graphic': boolean; + }; + category_scores: { + 'hate': number; + 'hate/threatening': number; + 'self-harm': number; + 'sexual': number; + 'sexual/minors': number; + 'violence': number; + 'violence/graphic': number; + }; + flagged: boolean; + }[]; +} + +/** + * Callback for fine control over the policy. It contains the event and the OpenAI moderation data. + * Implementations should return `true` to **reject** the content, and `false` to accept. + */ +type OpenAIHandler = (event: Event, data: ModerationData) => boolean; + +/** Policy options for `openaiPolicy`. */ +interface OpenAI { + handler?: OpenAIHandler; + endpoint?: string; + apiKey?: string; +} + +/** Default handler. Simply checks whether OpenAI flagged the content. */ +const flaggedHandler: OpenAIHandler = (_, { results }) => results.some((r) => r.flagged); + +/** + * Passes event content to OpenAI and then rejects flagged events. + * + * By default, this policy will reject kind 1 events that OpenAI flags. + * It's possible to pass a custom handler for more control. An OpenAI API key is required. + * + * @example + * ```ts + * // Default handler. It's so strict it's suitable for school children. + * openaiPolicy(msg, { apiKey: Deno.env.get('OPENAI_API_KEY') }); + * + * // With a custom handler. + * openaiPolicy(msg, { + * apiKey: Deno.env.get('OPENAI_API_KEY'), + * handler(event, result) { + * // Loop each result. + * return data.results.some((result) => { + * if (result.flagged) { + * const { sexual, violence } = result.categories; + * // Reject only events flagged as sexual and violent. + * return sexual && violence; + * } + * }); + * }, + * }); + * ``` + */ +const openaiPolicy: Policy = async ({ event }, opts = {}) => { + const { + handler = flaggedHandler, + endpoint = 'https://api.openai.com/v1/moderations', + apiKey, + } = opts; + + if (event.kind === 1) { + const resp = await fetch(endpoint, { + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + input: event.content, + }), + }); + + const result = await resp.json(); + + if (handler(event, result)) { + return { + id: event.id, + action: 'reject', + msg: 'blocked: content flagged by AI', + }; + } + } + + return { + id: event.id, + action: 'accept', + msg: '', + }; +}; + +export { flaggedHandler, openaiPolicy as default }; + +export type { ModerationData, OpenAI, OpenAIHandler };