diff --git a/README.md b/README.md index 774a018..fffd7d8 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,30 @@ Create a SQLite database `pubkeys.db` from the old csv file of users. This is id ```sh deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts +deno run -A fetch.ts +deno run -A fetch.ts --legacy-mode ``` +The legacy mode flag disables outbox mode to workaround issues where outbox mode stalls indefinitely on some profiles. + +Moreover, the websocket implementation in Deno will frequently crash for whatever reason. This means we need to run the script many times before it finishes collecting npubs. + +There's no update method, so if you want to update the foaf pubkey lists, delete `pubkey.db` and run the script again. + ## Development -Use repl: +Using repl: ```shell export DEBUG='ndk:*' deno repl -A ``` -Paste code. +Paste code from `fetch.ts`. Profit. + +TODO: + +* rename pubkey to subscriber_pubkey for consitency. ## Reference diff --git a/SQL.md b/SQL.md new file mode 100644 index 0000000..d753ed4 --- /dev/null +++ b/SQL.md @@ -0,0 +1,17 @@ +# sql + +Who has the most follows? + +```sql +SELECT subscriber_pubkey, COUNT(foaf_pubkey) AS foaf_count + FROM foaf + GROUP BY subscriber_pubkey + ORDER BY foaf_count DESC + LIMIT 10; +``` + +Count of all foafs + +```sql +SELECT COUNT(*) as foaf_count FROM foaf; +``` \ No newline at end of file diff --git a/copypasta-read-csv.ts b/copypasta-read-csv.ts new file mode 100644 index 0000000..55b5dbe --- /dev/null +++ b/copypasta-read-csv.ts @@ -0,0 +1,15 @@ + +import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts"; + +async function readCsv(filepath: string) { + const fileContents = await Deno.readTextFile(filepath); + const result = await parse(fileContents, { + skipFirstRow: true, // set to false if you want to include the header + columns: ["pubkey", "balance", "updated_at"], // define the columns if needed + }); + + return result; +} + +const filepath = "subscribers.csv"; +readCsv(filepath).then((data) => console.log(data)); \ No newline at end of file diff --git a/fetch.ts b/fetch.ts new file mode 100644 index 0000000..689edc0 --- /dev/null +++ b/fetch.ts @@ -0,0 +1,134 @@ +import NDK, { NDKUserProfile, NDKUser } from 'npm:@nostr-dev-kit/ndk'; + +import { DB } from "https://deno.land/x/sqlite@v3.8/mod.ts"; + +const TIMEOUT_MS = 5000; // 5 seconds timeout for each subscriber + +const db = new DB("pubkeys.db"); + +const legacyMode = Deno.args.includes("--legacy-mode"); + +const ndk = new NDK({ + explicitRelayUrls: [ + "wss://offchain.pub", + "wss://relay.primal.net", + "wss://relay.snort.social", + "wss://nos.lol", + ], + outboxRelayUrls: [ + "wss://bitcoiner.social", + "wss://welcome.nostr.wine", + ], + enableOutboxModel: !legacyMode, // Default is true, unless legacy mode is enabled +}); + +function insertFoaf(subscriberPubkey: string, foafPubkey: string) { + try { + db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]); + } catch (error) { + return; // do nothing + } +} + +async function getName(user: NDKUser): Promise { + await user.fetchProfile(); + if (user.profile) { + const profile: NDKUserProfile = user.profile; + return profile.nip05 || profile.name || profile.username || user.npub; + } else { + return user.npub; + } +} + +function timeout(ms: number) { + return new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms)); +} + +async function processSubscriber(subscriberPubkey: string, subscriber: NDKUser, TIMEOUT_MS: number) { + try { + const subscriberName = await Promise.race([getName(subscriber), timeout(TIMEOUT_MS)]); + if (typeof subscriberName !== 'string') { + throw new Error('Timeout occurred while fetching subscriber name.'); + } + + const followsResult = await Promise.race([subscriber.follows(), timeout(TIMEOUT_MS)]); + if (!(followsResult instanceof Set)) { + throw new Error('Timeout occurred while fetching follows.'); + } + + if (followsResult.size === 0) { + db.query("UPDATE subscribers SET has_no_foaf = 1 WHERE pubkey = ?", [subscriberPubkey]); + console.log(`Success: ${subscriberName} follows no one`); + return; + } + + for (const foaf of followsResult) { + insertFoaf(subscriberPubkey, foaf.pubkey); + } + + const foafCount = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number; + if (foafCount === followsResult.size) { + console.log(`Success: ${subscriberName} follows ${followsResult.size}`); + } else { + console.log(`Warning: ${subscriberName} follows ${followsResult.size} and ${foafCount} are in the database`); + } + } catch (error) { + if (error.message === 'Timeout') { + console.log(`Operation timed out for subscriber ${subscriberPubkey}. Continuing to next subscriber.`); + } else { + console.error("Error in processing subscriber:", error); + } + } +} + +async function processSubscribers(subscriberList: string[]) { + //const skipList = [ + // "ce2fb8588e047b61e738bee312bf63e03f9c1fd849ab67ab4c5f9b39643d5ffd" + //]; + for (const subscriberResult of subscriberList) { + const subscriberPubkey = subscriberResult as string; + //if (skipList.includes(subscriberPubkey)) { + // console.log(`Skipping: ${subscriberPubkey} is in skip list`); + // continue; + //} + const subscriber = ndk.getUser({ pubkey: subscriberPubkey }); + const foafCountPreCheck = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number; + const hasNoFoaf = db.query("SELECT has_no_foaf FROM subscribers WHERE pubkey = ?", [subscriberPubkey])[0][0] as boolean; + if (foafCountPreCheck > 0 || hasNoFoaf) { + console.log(`Skipping: ${subscriber.npub} already has ${foafCountPreCheck} follows`); + continue; + } + console.log("subscriber:", subscriber.pubkey) + await processSubscriber(subscriberPubkey, subscriber, TIMEOUT_MS); + } +} + +async function main() { + + await ndk.connect(); + + db.query(` + CREATE TABLE IF NOT EXISTS foaf ( + foaf_pubkey TEXT, + subscriber_pubkey TEXT, + UNIQUE(foaf_pubkey, subscriber_pubkey) + ); + `); + + //const subscriberResults = db.query("SELECT pubkey FROM subscribers ORDER BY pubkey DESC") as string[][]; + const subscriberResults = db.query(` + SELECT s.pubkey + FROM subscribers s + LEFT JOIN foaf f ON s.pubkey = f.subscriber_pubkey + WHERE s.has_no_foaf = 0 AND f.foaf_pubkey IS NULL + GROUP BY s.pubkey + `) as string[][]; + + const subscriberList = subscriberResults.map(subscriber => subscriber[0]); + + await processSubscribers(subscriberList); + db.close(); + Deno.exit(); +} + +main(); \ No newline at end of file diff --git a/initialize-subscriber-table.ts b/initialize-subscriber-table.ts new file mode 100644 index 0000000..f304c32 --- /dev/null +++ b/initialize-subscriber-table.ts @@ -0,0 +1,33 @@ +import { DB } from "https://deno.land/x/sqlite/mod.ts"; +import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts"; + +async function readCSV(filepath: string) { + const fileContents = await Deno.readTextFile(filepath); + const result = await parse(fileContents, { + skipFirstRow: true, + columns: ["pubkey", "balance", "updated_at"], + }); + + return result; +} + +function convertToUnixTimestamp(dateStr: string): number { + const date = new Date(dateStr); + return Math.floor(date.getTime() / 1000); +} +const db = new DB("pubkeys.db"); +db.query(` + CREATE TABLE IF NOT EXISTS subscribers ( + pubkey TEXT PRIMARY KEY, + created_at INTEGER, + balance INTEGER, + has_no_foaf INTEGER DEFAULT 0 + ) +`); +readCSV("subscribers.csv").then((subscribers) => { + for (const subscriber of subscribers) { + const createdAt = convertToUnixTimestamp(subscriber["updated_at"]); + db.query("INSERT OR IGNORE INTO subscribers (pubkey, balance, created_at) VALUES (?, ?, ?)", [subscriber["pubkey"], subscriber["balance"], createdAt]); + } + db.close(); +}); diff --git a/scratchpad.ts b/scratchpad.ts deleted file mode 100644 index 216bdce..0000000 --- a/scratchpad.ts +++ /dev/null @@ -1,48 +0,0 @@ -//const filter: NDKFilter = { kinds: [3], authors: ["69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d"] }; -//const event = await ndk.fetchEvent(filter); - -// https://ndk.fyi/docs/classes/NDKEvent.html -//console.log(event.rawEvent()); -//console.log(event?.content); -//console.log(event?.created_at); -//console.log(event?.tags.length); - -//event.rawEvent().tags.forEach((tag) => { -// if (tag[0] === "p"){ -// console.log(tag[1]) -// } -//}); - -async function insertFoaf(subscriberPubkey: string, foafPubkey: string) { - db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]); -} - - -const filter: NDKFilter = { kinds: [3], authors: [subscriberPubkey] }; -const follows = await ndk.fetchEvent(filter); - -follows.tags.forEach((tag) => { - if (tag[0] === "p") { - const foafPubkey = tag[1]; // Extract the FOAF pubkey - console.log(foafPubkey); // Log the FOAF pubkey - - // Insert the FOAF pubkey into the database - insertFoaf(subscriber_pubkey, foafPubkey); - } -}); - -// npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st -// hex: 69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d -const blee = ndk.getUser({ - npub: "npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st", -}); -// https://ndk.fyi/docs/classes/NDKUser.html#follows -const follows = await blee.follows(); -console.log("Number of follows:", follows.size); - - -for (const foaf of follows) { - console.log(foaf.pubkey); -} -// Will return only the first event -event = await ndk.fetchEvent(filter);