blee/npubweb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First collection completed.

Browse Source
This commit is contained in:
Brian Lee 2024-01-12 13:33:20 -08:00
parent cf3e3a7175
commit 6dea76fdc7
6 changed files with 213 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -8,18 +8,30 @@ Create a SQLite database `pubkeys.db` from the old csv file of users. This is id
```sh ```sh
deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts
deno run -A fetch.ts
deno run -A fetch.ts --legacy-mode
``` ```
The legacy mode flag disables outbox mode to workaround issues where outbox mode stalls indefinitely on some profiles.
Moreover, the websocket implementation in Deno will frequently crash for whatever reason. This means we need to run the script many times before it finishes collecting npubs.
There's no update method, so if you want to update the foaf pubkey lists, delete `pubkey.db` and run the script again.
## Development ## Development
Use repl: Using repl:
```shell ```shell
export DEBUG='ndk:*' export DEBUG='ndk:*'
deno repl -A deno repl -A
``` ```
Paste code. Paste code from `fetch.ts`. Profit.
TODO:
* rename pubkey to subscriber_pubkey for consitency.
## Reference ## Reference

17
SQL.md Normal file
View File

@ -0,0 +1,17 @@
# sql
Who has the most follows?
```sql
SELECT subscriber_pubkey, COUNT(foaf_pubkey) AS foaf_count
FROM foaf
GROUP BY subscriber_pubkey
ORDER BY foaf_count DESC
LIMIT 10;
```
Count of all foafs
```sql
SELECT COUNT(*) as foaf_count FROM foaf;
```

15
copypasta-read-csv.ts Normal file
View File

@ -0,0 +1,15 @@
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
async function readCsv(filepath: string) {
const fileContents = await Deno.readTextFile(filepath);
const result = await parse(fileContents, {
skipFirstRow: true, // set to false if you want to include the header
columns: ["pubkey", "balance", "updated_at"], // define the columns if needed
});
return result;
}
const filepath = "subscribers.csv";
readCsv(filepath).then((data) => console.log(data));

134
fetch.ts Normal file
View File

@ -0,0 +1,134 @@
import NDK, { NDKUserProfile, NDKUser } from 'npm:@nostr-dev-kit/ndk';
import { DB } from "https://deno.land/x/sqlite@v3.8/mod.ts";
const TIMEOUT_MS = 5000; // 5 seconds timeout for each subscriber
const db = new DB("pubkeys.db");
const legacyMode = Deno.args.includes("--legacy-mode");
const ndk = new NDK({
explicitRelayUrls: [
"wss://offchain.pub",
"wss://relay.primal.net",
"wss://relay.snort.social",
"wss://nos.lol",
],
outboxRelayUrls: [
"wss://bitcoiner.social",
"wss://welcome.nostr.wine",
],
enableOutboxModel: !legacyMode, // Default is true, unless legacy mode is enabled
});
function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
try {
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
} catch (error) {
return; // do nothing
}
}
async function getName(user: NDKUser): Promise<string> {
await user.fetchProfile();
if (user.profile) {
const profile: NDKUserProfile = user.profile;
return profile.nip05 || profile.name || profile.username || user.npub;
} else {
return user.npub;
}
}
function timeout(ms: number) {
return new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms));
}
async function processSubscriber(subscriberPubkey: string, subscriber: NDKUser, TIMEOUT_MS: number) {
try {
const subscriberName = await Promise.race([getName(subscriber), timeout(TIMEOUT_MS)]);
if (typeof subscriberName !== 'string') {
throw new Error('Timeout occurred while fetching subscriber name.');
}
const followsResult = await Promise.race([subscriber.follows(), timeout(TIMEOUT_MS)]);
if (!(followsResult instanceof Set)) {
throw new Error('Timeout occurred while fetching follows.');
}
if (followsResult.size === 0) {
db.query("UPDATE subscribers SET has_no_foaf = 1 WHERE pubkey = ?", [subscriberPubkey]);
console.log(`Success: ${subscriberName} follows no one`);
return;
}
for (const foaf of followsResult) {
insertFoaf(subscriberPubkey, foaf.pubkey);
}
const foafCount = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
if (foafCount === followsResult.size) {
console.log(`Success: ${subscriberName} follows ${followsResult.size}`);
} else {
console.log(`Warning: ${subscriberName} follows ${followsResult.size} and ${foafCount} are in the database`);
}
} catch (error) {
if (error.message === 'Timeout') {
console.log(`Operation timed out for subscriber ${subscriberPubkey}. Continuing to next subscriber.`);
} else {
console.error("Error in processing subscriber:", error);
}
}
}
async function processSubscribers(subscriberList: string[]) {
//const skipList = [
// "ce2fb8588e047b61e738bee312bf63e03f9c1fd849ab67ab4c5f9b39643d5ffd"
//];
for (const subscriberResult of subscriberList) {
const subscriberPubkey = subscriberResult as string;
//if (skipList.includes(subscriberPubkey)) {
// console.log(`Skipping: ${subscriberPubkey} is in skip list`);
// continue;
//}
const subscriber = ndk.getUser({ pubkey: subscriberPubkey });
const foafCountPreCheck = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
const hasNoFoaf = db.query("SELECT has_no_foaf FROM subscribers WHERE pubkey = ?", [subscriberPubkey])[0][0] as boolean;
if (foafCountPreCheck > 0 || hasNoFoaf) {
console.log(`Skipping: ${subscriber.npub} already has ${foafCountPreCheck} follows`);
continue;
}
console.log("subscriber:", subscriber.pubkey)
await processSubscriber(subscriberPubkey, subscriber, TIMEOUT_MS);
}
}
async function main() {
await ndk.connect();
db.query(`
CREATE TABLE IF NOT EXISTS foaf (
foaf_pubkey TEXT,
subscriber_pubkey TEXT,
UNIQUE(foaf_pubkey, subscriber_pubkey)
);
`);
//const subscriberResults = db.query("SELECT pubkey FROM subscribers ORDER BY pubkey DESC") as string[][];
const subscriberResults = db.query(`
SELECT s.pubkey
FROM subscribers s
LEFT JOIN foaf f ON s.pubkey = f.subscriber_pubkey
WHERE s.has_no_foaf = 0 AND f.foaf_pubkey IS NULL
GROUP BY s.pubkey
`) as string[][];
const subscriberList = subscriberResults.map(subscriber => subscriber[0]);
await processSubscribers(subscriberList);
db.close();
Deno.exit();
}
main();

33
initialize-subscriber-table.ts Normal file
View File

@ -0,0 +1,33 @@
import { DB } from "https://deno.land/x/sqlite/mod.ts";
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
async function readCSV(filepath: string) {
const fileContents = await Deno.readTextFile(filepath);
const result = await parse(fileContents, {
skipFirstRow: true,
columns: ["pubkey", "balance", "updated_at"],
});
return result;
}
function convertToUnixTimestamp(dateStr: string): number {
const date = new Date(dateStr);
return Math.floor(date.getTime() / 1000);
}
const db = new DB("pubkeys.db");
db.query(`
CREATE TABLE IF NOT EXISTS subscribers (
pubkey TEXT PRIMARY KEY,
created_at INTEGER,
balance INTEGER,
has_no_foaf INTEGER DEFAULT 0
)
`);
readCSV("subscribers.csv").then((subscribers) => {
for (const subscriber of subscribers) {
const createdAt = convertToUnixTimestamp(subscriber["updated_at"]);
db.query("INSERT OR IGNORE INTO subscribers (pubkey, balance, created_at) VALUES (?, ?, ?)", [subscriber["pubkey"], subscriber["balance"], createdAt]);
}
db.close();
});

48
scratchpad.ts
View File

@ -1,48 +0,0 @@
//const filter: NDKFilter = { kinds: [3], authors: ["69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d"] };
//const event = await ndk.fetchEvent(filter);
// https://ndk.fyi/docs/classes/NDKEvent.html
//console.log(event.rawEvent());
//console.log(event?.content);
//console.log(event?.created_at);
//console.log(event?.tags.length);
//event.rawEvent().tags.forEach((tag) => {
// if (tag[0] === "p"){
// console.log(tag[1])
// }
//});
async function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
}
const filter: NDKFilter = { kinds: [3], authors: [subscriberPubkey] };
const follows = await ndk.fetchEvent(filter);
follows.tags.forEach((tag) => {
if (tag[0] === "p") {
const foafPubkey = tag[1]; // Extract the FOAF pubkey
console.log(foafPubkey); // Log the FOAF pubkey
// Insert the FOAF pubkey into the database
insertFoaf(subscriber_pubkey, foafPubkey);
}
});
// npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st
// hex: 69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d
const blee = ndk.getUser({
npub: "npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st",
});
// https://ndk.fyi/docs/classes/NDKUser.html#follows
const follows = await blee.follows();
console.log("Number of follows:", follows.size);
for (const foaf of follows) {
console.log(foaf.pubkey);
}
// Will return only the first event
event = await ndk.fetchEvent(filter);