First collection completed.
This commit is contained in:
parent
cf3e3a7175
commit
6dea76fdc7
16
README.md
16
README.md
@ -8,18 +8,30 @@ Create a SQLite database `pubkeys.db` from the old csv file of users. This is id
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts
|
deno run --allow-read=. --allow-write=. initialize-subscriber-table.ts
|
||||||
|
deno run -A fetch.ts
|
||||||
|
deno run -A fetch.ts --legacy-mode
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The legacy mode flag disables outbox mode to workaround issues where outbox mode stalls indefinitely on some profiles.
|
||||||
|
|
||||||
|
Moreover, the websocket implementation in Deno will frequently crash for whatever reason. This means we need to run the script many times before it finishes collecting npubs.
|
||||||
|
|
||||||
|
There's no update method, so if you want to update the foaf pubkey lists, delete `pubkey.db` and run the script again.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
Use repl:
|
Using repl:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
export DEBUG='ndk:*'
|
export DEBUG='ndk:*'
|
||||||
deno repl -A
|
deno repl -A
|
||||||
```
|
```
|
||||||
|
|
||||||
Paste code.
|
Paste code from `fetch.ts`. Profit.
|
||||||
|
|
||||||
|
TODO:
|
||||||
|
|
||||||
|
* rename pubkey to subscriber_pubkey for consitency.
|
||||||
|
|
||||||
## Reference
|
## Reference
|
||||||
|
|
||||||
|
17
SQL.md
Normal file
17
SQL.md
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# sql
|
||||||
|
|
||||||
|
Who has the most follows?
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT subscriber_pubkey, COUNT(foaf_pubkey) AS foaf_count
|
||||||
|
FROM foaf
|
||||||
|
GROUP BY subscriber_pubkey
|
||||||
|
ORDER BY foaf_count DESC
|
||||||
|
LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
Count of all foafs
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT COUNT(*) as foaf_count FROM foaf;
|
||||||
|
```
|
15
copypasta-read-csv.ts
Normal file
15
copypasta-read-csv.ts
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
|
||||||
|
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
|
||||||
|
|
||||||
|
async function readCsv(filepath: string) {
|
||||||
|
const fileContents = await Deno.readTextFile(filepath);
|
||||||
|
const result = await parse(fileContents, {
|
||||||
|
skipFirstRow: true, // set to false if you want to include the header
|
||||||
|
columns: ["pubkey", "balance", "updated_at"], // define the columns if needed
|
||||||
|
});
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const filepath = "subscribers.csv";
|
||||||
|
readCsv(filepath).then((data) => console.log(data));
|
134
fetch.ts
Normal file
134
fetch.ts
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
import NDK, { NDKUserProfile, NDKUser } from 'npm:@nostr-dev-kit/ndk';
|
||||||
|
|
||||||
|
import { DB } from "https://deno.land/x/sqlite@v3.8/mod.ts";
|
||||||
|
|
||||||
|
const TIMEOUT_MS = 5000; // 5 seconds timeout for each subscriber
|
||||||
|
|
||||||
|
const db = new DB("pubkeys.db");
|
||||||
|
|
||||||
|
const legacyMode = Deno.args.includes("--legacy-mode");
|
||||||
|
|
||||||
|
const ndk = new NDK({
|
||||||
|
explicitRelayUrls: [
|
||||||
|
"wss://offchain.pub",
|
||||||
|
"wss://relay.primal.net",
|
||||||
|
"wss://relay.snort.social",
|
||||||
|
"wss://nos.lol",
|
||||||
|
],
|
||||||
|
outboxRelayUrls: [
|
||||||
|
"wss://bitcoiner.social",
|
||||||
|
"wss://welcome.nostr.wine",
|
||||||
|
],
|
||||||
|
enableOutboxModel: !legacyMode, // Default is true, unless legacy mode is enabled
|
||||||
|
});
|
||||||
|
|
||||||
|
function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
|
||||||
|
try {
|
||||||
|
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
|
||||||
|
} catch (error) {
|
||||||
|
return; // do nothing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getName(user: NDKUser): Promise<string> {
|
||||||
|
await user.fetchProfile();
|
||||||
|
if (user.profile) {
|
||||||
|
const profile: NDKUserProfile = user.profile;
|
||||||
|
return profile.nip05 || profile.name || profile.username || user.npub;
|
||||||
|
} else {
|
||||||
|
return user.npub;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function timeout(ms: number) {
|
||||||
|
return new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processSubscriber(subscriberPubkey: string, subscriber: NDKUser, TIMEOUT_MS: number) {
|
||||||
|
try {
|
||||||
|
const subscriberName = await Promise.race([getName(subscriber), timeout(TIMEOUT_MS)]);
|
||||||
|
if (typeof subscriberName !== 'string') {
|
||||||
|
throw new Error('Timeout occurred while fetching subscriber name.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const followsResult = await Promise.race([subscriber.follows(), timeout(TIMEOUT_MS)]);
|
||||||
|
if (!(followsResult instanceof Set)) {
|
||||||
|
throw new Error('Timeout occurred while fetching follows.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (followsResult.size === 0) {
|
||||||
|
db.query("UPDATE subscribers SET has_no_foaf = 1 WHERE pubkey = ?", [subscriberPubkey]);
|
||||||
|
console.log(`Success: ${subscriberName} follows no one`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const foaf of followsResult) {
|
||||||
|
insertFoaf(subscriberPubkey, foaf.pubkey);
|
||||||
|
}
|
||||||
|
|
||||||
|
const foafCount = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
|
||||||
|
if (foafCount === followsResult.size) {
|
||||||
|
console.log(`Success: ${subscriberName} follows ${followsResult.size}`);
|
||||||
|
} else {
|
||||||
|
console.log(`Warning: ${subscriberName} follows ${followsResult.size} and ${foafCount} are in the database`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error.message === 'Timeout') {
|
||||||
|
console.log(`Operation timed out for subscriber ${subscriberPubkey}. Continuing to next subscriber.`);
|
||||||
|
} else {
|
||||||
|
console.error("Error in processing subscriber:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processSubscribers(subscriberList: string[]) {
|
||||||
|
//const skipList = [
|
||||||
|
// "ce2fb8588e047b61e738bee312bf63e03f9c1fd849ab67ab4c5f9b39643d5ffd"
|
||||||
|
//];
|
||||||
|
for (const subscriberResult of subscriberList) {
|
||||||
|
const subscriberPubkey = subscriberResult as string;
|
||||||
|
//if (skipList.includes(subscriberPubkey)) {
|
||||||
|
// console.log(`Skipping: ${subscriberPubkey} is in skip list`);
|
||||||
|
// continue;
|
||||||
|
//}
|
||||||
|
const subscriber = ndk.getUser({ pubkey: subscriberPubkey });
|
||||||
|
const foafCountPreCheck = db.query("SELECT COUNT(*) FROM foaf WHERE subscriber_pubkey = ?", [subscriberPubkey])[0][0] as number;
|
||||||
|
const hasNoFoaf = db.query("SELECT has_no_foaf FROM subscribers WHERE pubkey = ?", [subscriberPubkey])[0][0] as boolean;
|
||||||
|
if (foafCountPreCheck > 0 || hasNoFoaf) {
|
||||||
|
console.log(`Skipping: ${subscriber.npub} already has ${foafCountPreCheck} follows`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
console.log("subscriber:", subscriber.pubkey)
|
||||||
|
await processSubscriber(subscriberPubkey, subscriber, TIMEOUT_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
|
||||||
|
await ndk.connect();
|
||||||
|
|
||||||
|
db.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS foaf (
|
||||||
|
foaf_pubkey TEXT,
|
||||||
|
subscriber_pubkey TEXT,
|
||||||
|
UNIQUE(foaf_pubkey, subscriber_pubkey)
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
//const subscriberResults = db.query("SELECT pubkey FROM subscribers ORDER BY pubkey DESC") as string[][];
|
||||||
|
const subscriberResults = db.query(`
|
||||||
|
SELECT s.pubkey
|
||||||
|
FROM subscribers s
|
||||||
|
LEFT JOIN foaf f ON s.pubkey = f.subscriber_pubkey
|
||||||
|
WHERE s.has_no_foaf = 0 AND f.foaf_pubkey IS NULL
|
||||||
|
GROUP BY s.pubkey
|
||||||
|
`) as string[][];
|
||||||
|
|
||||||
|
const subscriberList = subscriberResults.map(subscriber => subscriber[0]);
|
||||||
|
|
||||||
|
await processSubscribers(subscriberList);
|
||||||
|
db.close();
|
||||||
|
Deno.exit();
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
33
initialize-subscriber-table.ts
Normal file
33
initialize-subscriber-table.ts
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
import { DB } from "https://deno.land/x/sqlite/mod.ts";
|
||||||
|
import { parse } from "https://deno.land/std@0.211.0/csv/mod.ts";
|
||||||
|
|
||||||
|
async function readCSV(filepath: string) {
|
||||||
|
const fileContents = await Deno.readTextFile(filepath);
|
||||||
|
const result = await parse(fileContents, {
|
||||||
|
skipFirstRow: true,
|
||||||
|
columns: ["pubkey", "balance", "updated_at"],
|
||||||
|
});
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function convertToUnixTimestamp(dateStr: string): number {
|
||||||
|
const date = new Date(dateStr);
|
||||||
|
return Math.floor(date.getTime() / 1000);
|
||||||
|
}
|
||||||
|
const db = new DB("pubkeys.db");
|
||||||
|
db.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS subscribers (
|
||||||
|
pubkey TEXT PRIMARY KEY,
|
||||||
|
created_at INTEGER,
|
||||||
|
balance INTEGER,
|
||||||
|
has_no_foaf INTEGER DEFAULT 0
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
readCSV("subscribers.csv").then((subscribers) => {
|
||||||
|
for (const subscriber of subscribers) {
|
||||||
|
const createdAt = convertToUnixTimestamp(subscriber["updated_at"]);
|
||||||
|
db.query("INSERT OR IGNORE INTO subscribers (pubkey, balance, created_at) VALUES (?, ?, ?)", [subscriber["pubkey"], subscriber["balance"], createdAt]);
|
||||||
|
}
|
||||||
|
db.close();
|
||||||
|
});
|
@ -1,48 +0,0 @@
|
|||||||
//const filter: NDKFilter = { kinds: [3], authors: ["69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d"] };
|
|
||||||
//const event = await ndk.fetchEvent(filter);
|
|
||||||
|
|
||||||
// https://ndk.fyi/docs/classes/NDKEvent.html
|
|
||||||
//console.log(event.rawEvent());
|
|
||||||
//console.log(event?.content);
|
|
||||||
//console.log(event?.created_at);
|
|
||||||
//console.log(event?.tags.length);
|
|
||||||
|
|
||||||
//event.rawEvent().tags.forEach((tag) => {
|
|
||||||
// if (tag[0] === "p"){
|
|
||||||
// console.log(tag[1])
|
|
||||||
// }
|
|
||||||
//});
|
|
||||||
|
|
||||||
async function insertFoaf(subscriberPubkey: string, foafPubkey: string) {
|
|
||||||
db.query("INSERT INTO foaf (foaf_pubkey, subscriber_pubkey) VALUES (?, ?)", [foafPubkey, subscriberPubkey]);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
const filter: NDKFilter = { kinds: [3], authors: [subscriberPubkey] };
|
|
||||||
const follows = await ndk.fetchEvent(filter);
|
|
||||||
|
|
||||||
follows.tags.forEach((tag) => {
|
|
||||||
if (tag[0] === "p") {
|
|
||||||
const foafPubkey = tag[1]; // Extract the FOAF pubkey
|
|
||||||
console.log(foafPubkey); // Log the FOAF pubkey
|
|
||||||
|
|
||||||
// Insert the FOAF pubkey into the database
|
|
||||||
insertFoaf(subscriber_pubkey, foafPubkey);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st
|
|
||||||
// hex: 69a0a0910b49a1dbfbc4e4f10df22b5806af5403a228267638f2e908c968228d
|
|
||||||
const blee = ndk.getUser({
|
|
||||||
npub: "npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st",
|
|
||||||
});
|
|
||||||
// https://ndk.fyi/docs/classes/NDKUser.html#follows
|
|
||||||
const follows = await blee.follows();
|
|
||||||
console.log("Number of follows:", follows.size);
|
|
||||||
|
|
||||||
|
|
||||||
for (const foaf of follows) {
|
|
||||||
console.log(foaf.pubkey);
|
|
||||||
}
|
|
||||||
// Will return only the first event
|
|
||||||
event = await ndk.fetchEvent(filter);
|
|
Loading…
Reference in New Issue
Block a user