npubweb/fetch.ts

import NDK, { NDKUserProfile, NDKUser } from 'npm:@nostr-dev-kit/ndk';

import { DB } from "https://deno.land/x/sqlite@v3.8/mod.ts";

const TIMEOUT_MS = 5000; // 5 seconds timeout for each subscriber

const db = new DB("pubkeys.db");

const legacyMode = Deno.args.includes("--legacy-mode");

const ndk = new NDK({
    explicitRelayUrls: [
        "wss://offchain.pub",
        "wss://relay.primal.net",
        "wss://relay.snort.social",
        "wss://nos.lol",
    ],
    outboxRelayUrls: [
        "wss://bitcoiner.social",
        "wss://welcome.nostr.wine",
    ],
    enableOutboxModel: !legacyMode, // Default is true, unless legacy mode is enabled
});

function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
    try {
        db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
    } catch (error) {
        return; // do nothing
    }   
}

async function getName(user: NDKUser): Promise<string> {
    await user.fetchProfile();
    if (user.profile) {
        const profile: NDKUserProfile = user.profile;
        return profile.nip05 || profile.name || profile.username || user.npub;
    } else {
        return user.npub;
    }
}

function timeout(ms: number) {
    return new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms));
}

async function processSubscriber(subscriberPubkey: string, subscriber: NDKUser, TIMEOUT_MS: number) {
    try {
        const subscriberName = await Promise.race([getName(subscriber), timeout(TIMEOUT_MS)]);
        if (typeof subscriberName !== 'string') {
            throw new Error('Timeout occurred while fetching subscriber name.');
        }

        const followsResult = await Promise.race([subscriber.follows(), timeout(TIMEOUT_MS)]);
        if (!(followsResult instanceof Set)) {
            throw new Error('Timeout occurred while fetching follows.');
        }

        if (followsResult.size === 0) {
            db.query("UPDATE subscribers SET has_no_foaf = 1 WHERE pubkey = ?", [subscriberPubkey]);
            console.log(`Success: ${subscriberName} follows no one`);
            return;
        }

        for (const foaf of followsResult) {
            insertFoaf(subscriberPubkey, foaf.pubkey);
        }

        const foafCount = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
        if (foafCount === followsResult.size) {
            console.log(`Success: ${subscriberName} follows ${followsResult.size}`);
        } else {
            console.log(`Warning: ${subscriberName} follows ${followsResult.size} and ${foafCount} are in the database`);
        }
    } catch (error) {
        if (error.message === 'Timeout') {
            console.log(`Operation timed out for subscriber ${subscriberPubkey}. Continuing to next subscriber.`);
        } else {
            console.error("Error in processing subscriber:", error);
        }
    }
}

async function processSubscribers(subscriberList: string[]) {
  //const skipList = [
  //    "ce2fb8588e047b61e738bee312bf63e03f9c1fd849ab67ab4c5f9b39643d5ffd"
  //];
    for (const subscriberResult of subscriberList) {
        const subscriberPubkey = subscriberResult as string;
      //if (skipList.includes(subscriberPubkey)) {
      //    console.log(`Skipping: ${subscriberPubkey} is in skip list`);
      //    continue;
      //}
        const subscriber = ndk.getUser({ pubkey: subscriberPubkey });
        const foafCountPreCheck = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
        const hasNoFoaf = db.query("SELECT has_no_foaf FROM subscribers WHERE pubkey = ?", [subscriberPubkey])[0][0] as boolean;
        if (foafCountPreCheck > 0 || hasNoFoaf) {
            console.log(`Skipping: ${subscriber.npub} already has ${foafCountPreCheck} follows`);
            continue;
        }
        console.log("subscriber:", subscriber.pubkey)
        await processSubscriber(subscriberPubkey, subscriber, TIMEOUT_MS);
    }
}

async function main() {

    await ndk.connect();

    db.query(`
        CREATE TABLE IF NOT EXISTS foaf (
            foaf_pubkey TEXT,
            subscriber_pubkey TEXT,
            UNIQUE(foaf_pubkey, subscriber_pubkey)
        );
    `);

    //const subscriberResults = db.query("SELECT pubkey FROM subscribers ORDER BY pubkey DESC") as string[][];
    const subscriberResults = db.query(`
        SELECT s.pubkey 
        FROM subscribers s
        LEFT JOIN foaf f ON s.pubkey = f.subscriber_pubkey
        WHERE s.has_no_foaf = 0 AND f.foaf_pubkey IS NULL
        GROUP BY s.pubkey
    `) as string[][];

    const subscriberList = subscriberResults.map(subscriber => subscriber[0]);

    await processSubscribers(subscriberList);
    db.close();
    Deno.exit();
}

main();