First collection completed.
This commit is contained in:
parent
cf3e3a7175
commit
6dea76fdc7
16
README.md
16
README.md
@ -8,18 +8,30 @@ Create a SQLite database `pubkeys.db` from the old csv file of users. This is id
|
||||
|
||||
```sh
|
||||
deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts
|
||||
deno run -A fetch.ts
|
||||
deno run -A fetch.ts --legacy-mode
|
||||
```
|
||||
|
||||
The legacy mode flag disables outbox mode to workaround issues where outbox mode stalls indefinitely on some profiles.
|
||||
|
||||
Moreover, the websocket implementation in Deno will frequently crash for whatever reason. This means we need to run the script many times before it finishes collecting npubs.
|
||||
|
||||
There's no update method, so if you want to update the foaf pubkey lists, delete `pubkey.db` and run the script again.
|
||||
|
||||
## Development
|
||||
|
||||
Use repl:
|
||||
Using repl:
|
||||
|
||||
```shell
|
||||
export DEBUG='ndk:*'
|
||||
deno repl -A
|
||||
```
|
||||
|
||||
Paste code.
|
||||
Paste code from `fetch.ts`. Profit.
|
||||
|
||||
TODO:
|
||||
|
||||
* rename pubkey to subscriber_pubkey for consitency.
|
||||
|
||||
## Reference
|
||||
|
||||
|
17
SQL.md
Normal file
17
SQL.md
Normal file
@ -0,0 +1,17 @@
|
||||
# sql
|
||||
|
||||
Who has the most follows?
|
||||
|
||||
```sql
|
||||
SELECT subscriber_pubkey, COUNT(foaf_pubkey) AS foaf_count
|
||||
FROM foaf
|
||||
GROUP BY subscriber_pubkey
|
||||
ORDER BY foaf_count DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
Count of all foafs
|
||||
|
||||
```sql
|
||||
SELECT COUNT(*) as foaf_count FROM foaf;
|
||||
```
|
15
copypasta-read-csv.ts
Normal file
15
copypasta-read-csv.ts
Normal file
@ -0,0 +1,15 @@
|
||||
|
||||
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
|
||||
|
||||
async function readCsv(filepath: string) {
|
||||
const fileContents = await Deno.readTextFile(filepath);
|
||||
const result = await parse(fileContents, {
|
||||
skipFirstRow: true, // set to false if you want to include the header
|
||||
columns: ["pubkey", "balance", "updated_at"], // define the columns if needed
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
const filepath = "subscribers.csv";
|
||||
readCsv(filepath).then((data) => console.log(data));
|
134
fetch.ts
Normal file
134
fetch.ts
Normal file
@ -0,0 +1,134 @@
|
||||
import NDK, { NDKUserProfile, NDKUser } from 'npm:@nostr-dev-kit/ndk';
|
||||
|
||||
import { DB } from "https://deno.land/x/sqlite@v3.8/mod.ts";
|
||||
|
||||
const TIMEOUT_MS = 5000; // 5 seconds timeout for each subscriber
|
||||
|
||||
const db = new DB("pubkeys.db");
|
||||
|
||||
const legacyMode = Deno.args.includes("--legacy-mode");
|
||||
|
||||
const ndk = new NDK({
|
||||
explicitRelayUrls: [
|
||||
"wss://offchain.pub",
|
||||
"wss://relay.primal.net",
|
||||
"wss://relay.snort.social",
|
||||
"wss://nos.lol",
|
||||
],
|
||||
outboxRelayUrls: [
|
||||
"wss://bitcoiner.social",
|
||||
"wss://welcome.nostr.wine",
|
||||
],
|
||||
enableOutboxModel: !legacyMode, // Default is true, unless legacy mode is enabled
|
||||
});
|
||||
|
||||
function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
|
||||
try {
|
||||
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
|
||||
} catch (error) {
|
||||
return; // do nothing
|
||||
}
|
||||
}
|
||||
|
||||
async function getName(user: NDKUser): Promise<string> {
|
||||
await user.fetchProfile();
|
||||
if (user.profile) {
|
||||
const profile: NDKUserProfile = user.profile;
|
||||
return profile.nip05 || profile.name || profile.username || user.npub;
|
||||
} else {
|
||||
return user.npub;
|
||||
}
|
||||
}
|
||||
|
||||
function timeout(ms: number) {
|
||||
return new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms));
|
||||
}
|
||||
|
||||
async function processSubscriber(subscriberPubkey: string, subscriber: NDKUser, TIMEOUT_MS: number) {
|
||||
try {
|
||||
const subscriberName = await Promise.race([getName(subscriber), timeout(TIMEOUT_MS)]);
|
||||
if (typeof subscriberName !== 'string') {
|
||||
throw new Error('Timeout occurred while fetching subscriber name.');
|
||||
}
|
||||
|
||||
const followsResult = await Promise.race([subscriber.follows(), timeout(TIMEOUT_MS)]);
|
||||
if (!(followsResult instanceof Set)) {
|
||||
throw new Error('Timeout occurred while fetching follows.');
|
||||
}
|
||||
|
||||
if (followsResult.size === 0) {
|
||||
db.query("UPDATE subscribers SET has_no_foaf = 1 WHERE pubkey = ?", [subscriberPubkey]);
|
||||
console.log(`Success: ${subscriberName} follows no one`);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const foaf of followsResult) {
|
||||
insertFoaf(subscriberPubkey, foaf.pubkey);
|
||||
}
|
||||
|
||||
const foafCount = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
|
||||
if (foafCount === followsResult.size) {
|
||||
console.log(`Success: ${subscriberName} follows ${followsResult.size}`);
|
||||
} else {
|
||||
console.log(`Warning: ${subscriberName} follows ${followsResult.size} and ${foafCount} are in the database`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.message === 'Timeout') {
|
||||
console.log(`Operation timed out for subscriber ${subscriberPubkey}. Continuing to next subscriber.`);
|
||||
} else {
|
||||
console.error("Error in processing subscriber:", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processSubscribers(subscriberList: string[]) {
|
||||
//const skipList = [
|
||||
// "ce2fb8588e047b61e738bee312bf63e03f9c1fd849ab67ab4c5f9b39643d5ffd"
|
||||
//];
|
||||
for (const subscriberResult of subscriberList) {
|
||||
const subscriberPubkey = subscriberResult as string;
|
||||
//if (skipList.includes(subscriberPubkey)) {
|
||||
// console.log(`Skipping: ${subscriberPubkey} is in skip list`);
|
||||
// continue;
|
||||
//}
|
||||
const subscriber = ndk.getUser({ pubkey: subscriberPubkey });
|
||||
const foafCountPreCheck = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
|
||||
const hasNoFoaf = db.query("SELECT has_no_foaf FROM subscribers WHERE pubkey = ?", [subscriberPubkey])[0][0] as boolean;
|
||||
if (foafCountPreCheck > 0 || hasNoFoaf) {
|
||||
console.log(`Skipping: ${subscriber.npub} already has ${foafCountPreCheck} follows`);
|
||||
continue;
|
||||
}
|
||||
console.log("subscriber:", subscriber.pubkey)
|
||||
await processSubscriber(subscriberPubkey, subscriber, TIMEOUT_MS);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
|
||||
await ndk.connect();
|
||||
|
||||
db.query(`
|
||||
CREATE TABLE IF NOT EXISTS foaf (
|
||||
foaf_pubkey TEXT,
|
||||
subscriber_pubkey TEXT,
|
||||
UNIQUE(foaf_pubkey, subscriber_pubkey)
|
||||
);
|
||||
`);
|
||||
|
||||
//const subscriberResults = db.query("SELECT pubkey FROM subscribers ORDER BY pubkey DESC") as string[][];
|
||||
const subscriberResults = db.query(`
|
||||
SELECT s.pubkey
|
||||
FROM subscribers s
|
||||
LEFT JOIN foaf f ON s.pubkey = f.subscriber_pubkey
|
||||
WHERE s.has_no_foaf = 0 AND f.foaf_pubkey IS NULL
|
||||
GROUP BY s.pubkey
|
||||
`) as string[][];
|
||||
|
||||
const subscriberList = subscriberResults.map(subscriber => subscriber[0]);
|
||||
|
||||
await processSubscribers(subscriberList);
|
||||
db.close();
|
||||
Deno.exit();
|
||||
}
|
||||
|
||||
main();
|
33
initialize-subscriber-table.ts
Normal file
33
initialize-subscriber-table.ts
Normal file
@ -0,0 +1,33 @@
|
||||
import { DB } from "https://deno.land/x/sqlite/mod.ts";
|
||||
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
|
||||
|
||||
async function readCSV(filepath: string) {
|
||||
const fileContents = await Deno.readTextFile(filepath);
|
||||
const result = await parse(fileContents, {
|
||||
skipFirstRow: true,
|
||||
columns: ["pubkey", "balance", "updated_at"],
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function convertToUnixTimestamp(dateStr: string): number {
|
||||
const date = new Date(dateStr);
|
||||
return Math.floor(date.getTime() / 1000);
|
||||
}
|
||||
const db = new DB("pubkeys.db");
|
||||
db.query(`
|
||||
CREATE TABLE IF NOT EXISTS subscribers (
|
||||
pubkey TEXT PRIMARY KEY,
|
||||
created_at INTEGER,
|
||||
balance INTEGER,
|
||||
has_no_foaf INTEGER DEFAULT 0
|
||||
)
|
||||
`);
|
||||
readCSV("subscribers.csv").then((subscribers) => {
|
||||
for (const subscriber of subscribers) {
|
||||
const createdAt = convertToUnixTimestamp(subscriber["updated_at"]);
|
||||
db.query("INSERT OR IGNORE INTO subscribers (pubkey, balance, created_at) VALUES (?, ?, ?)", [subscriber["pubkey"], subscriber["balance"], createdAt]);
|
||||
}
|
||||
db.close();
|
||||
});
|
@ -1,48 +0,0 @@
|
||||
//const filter: NDKFilter = { kinds: [3], authors: ["69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d"] };
|
||||
//const event = await ndk.fetchEvent(filter);
|
||||
|
||||
// https://ndk.fyi/docs/classes/NDKEvent.html
|
||||
//console.log(event.rawEvent());
|
||||
//console.log(event?.content);
|
||||
//console.log(event?.created_at);
|
||||
//console.log(event?.tags.length);
|
||||
|
||||
//event.rawEvent().tags.forEach((tag) => {
|
||||
// if (tag[0] === "p"){
|
||||
// console.log(tag[1])
|
||||
// }
|
||||
//});
|
||||
|
||||
async function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
|
||||
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
|
||||
}
|
||||
|
||||
|
||||
const filter: NDKFilter = { kinds: [3], authors: [subscriberPubkey] };
|
||||
const follows = await ndk.fetchEvent(filter);
|
||||
|
||||
follows.tags.forEach((tag) => {
|
||||
if (tag[0] === "p") {
|
||||
const foafPubkey = tag[1]; // Extract the FOAF pubkey
|
||||
console.log(foafPubkey); // Log the FOAF pubkey
|
||||
|
||||
// Insert the FOAF pubkey into the database
|
||||
insertFoaf(subscriber_pubkey, foafPubkey);
|
||||
}
|
||||
});
|
||||
|
||||
// npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st
|
||||
// hex: 69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d
|
||||
const blee = ndk.getUser({
|
||||
npub: "npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st",
|
||||
});
|
||||
// https://ndk.fyi/docs/classes/NDKUser.html#follows
|
||||
const follows = await blee.follows();
|
||||
console.log("Number of follows:", follows.size);
|
||||
|
||||
|
||||
for (const foaf of follows) {
|
||||
console.log(foaf.pubkey);
|
||||
}
|
||||
// Will return only the first event
|
||||
event = await ndk.fetchEvent(filter);
|
Loading…
Reference in New Issue
Block a user