global feed fixes

Author: arcbtc

src/lib/nostr/client.test.ts

@@ -131,6 +131,17 @@ describe('nostr client helpers', () => {
     ]);
   });
 
+  it('keeps trusted global authors through spam filtering unless muted', () => {
+    const trusted = 'a'.repeat(64);
+    const muted = new Set<string>();
+    const trustedSet = new Set([trusted]);
+    const noisy = event({ id: '1'.repeat(64), pubkey: trusted, content: `${'long '.repeat(300)} #adult` });
+    const regularNoisy = event({ id: '2'.repeat(64), pubkey: 'b'.repeat(64), content: `${'long '.repeat(300)} #adult` });
+
+    expect(filterSpam([noisy, regularNoisy], muted, trustedSet).map((item) => item.id)).toEqual([noisy.id]);
+    expect(filterSpam([noisy], new Set([trusted]), trustedSet)).toEqual([]);
+  });
+
   it('returns an empty follow or custom feed when there are no follows', async () => {
     await expect(fetchFeed('follow', [], [])).resolves.toEqual([]);
     await expect(fetchFeed('custom', [], [])).resolves.toEqual([]);
@@ -327,11 +338,20 @@ describe('nostr client helpers', () => {
     const root = event({ content: 'top-level note' });
     const markedReply = event({ content: 'reply', tags: [['e', root.id, 'wss://relay.example', 'reply', root.pubkey]] });
     const legacyReply = event({ content: 'legacy reply', tags: [['e', root.id]] });
+    const quoted = event({ content: 'quoted note', tags: [['e', root.id], ['q', root.id]] });
+    const mentioned = event({ content: 'mentioned note', tags: [['e', root.id, 'wss://relay.example', 'mention']] });
+    const embedded = event({
+      content: 'nostr:note1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j',
+      tags: [['e', root.id]]
+    });
 
     expect(isReplyEvent(root)).toBe(false);
     expect(isReplyEvent(markedReply)).toBe(true);
     expect(isReplyEvent(legacyReply)).toBe(true);
-    expect(topLevelFeedEvents([root, markedReply, legacyReply])).toEqual([root]);
+    expect(isReplyEvent(quoted)).toBe(false);
+    expect(isReplyEvent(mentioned)).toBe(false);
+    expect(isReplyEvent(embedded)).toBe(false);
+    expect(topLevelFeedEvents([root, markedReply, legacyReply, quoted, mentioned, embedded])).toEqual([root, quoted, mentioned, embedded]);
   });
 
   it('hides events only when deletion requests are signed by the original author', () => {

src/lib/nostr/client.ts

@@ -364,9 +364,10 @@ function pomegranateEmailFromToken(token: string) {
   }
 }
 
-export function filterSpam(events: NostrEvent[], mutedPubkeys = new Set<string>()) {
+export function filterSpam(events: NostrEvent[], mutedPubkeys = new Set<string>(), trustedPubkeys = new Set<string>()) {
   return events.filter((event) => {
     if (mutedPubkeys.has(event.pubkey)) return false;
+    if (trustedPubkeys.has(event.pubkey)) return true;
     if (event.kind === 6) return Boolean(parseRepostContent(event));
     if (event.kind === 1 && event.content.length > maxFeedPostContentLength) return false;
     if (isMachineGeneratedContent(event.content)) return false;
@@ -384,7 +385,11 @@ export function isReplyEvent(event: NostrEvent) {
   const eventTags = event.tags.filter((tag) => tag[0] === 'e' && tag[1]);
   if (!eventTags.length) return false;
 
-  return eventTags.some((tag) => tag[3] === 'root' || tag[3] === 'reply') || eventTags.length > 0;
+  if (eventTags.some((tag) => tag[3] === 'root' || tag[3] === 'reply')) return true;
+  if (eventTags.some((tag) => tag[3] === 'mention')) return false;
+  if (event.tags.some((tag) => tag[0] === 'q' && tag[1])) return false;
+  if (/\bnostr:(?:note|nevent)1[023456789acdefghjklmnpqrstuvwxyz]+/i.test(event.content)) return false;
+  return true;
 }
 
 export function isFamilySafeEvent(event: NostrEvent) {
@@ -609,16 +614,30 @@ export async function fetchFeed(
   if (options.until) base.until = options.until;
   const filters = await feedFiltersForMode(mode, base, follows, settings, since, relayUrls, options);
 
+  const trustedGlobalAuthors = mode === 'global' ? new Set(options.globalAuthors?.filter((pubkey) => /^[0-9a-f]{64}$/i.test(pubkey)) ?? []) : new Set<string>();
   const events = verifiedRelayEvents((await Promise.all(filters.map((filter) => queryShortLived(relayUrls, filter, 5000, { minEvents: feedIdleMinEvents(mode, filter.limit), idleAfterMs: 650 })))).flat()).filter((event) =>
     filters.some((filter) => eventMatchesTimeWindow(event, filter.since, filter.until))
   );
-  const clean = dedupeEvents(topLevelFeedEvents(filterSpam(events)));
-  const feedClean = mode === 'global' ? clean.filter((event) => !eventHasImgurUrl(event)) : clean;
-  const output = mode === 'global' ? limitCryptoTopicDensity(limitConsecutiveAuthors(feedClean, 2), 10) : feedClean;
+  const clean = dedupeEvents(topLevelFeedEvents(filterSpam(events, new Set<string>(), trustedGlobalAuthors)));
+  const feedClean = mode === 'global' ? clean.filter((event) => trustedGlobalAuthors.has(event.pubkey) || !eventHasImgurUrl(event)) : clean;
+  const output = mode === 'global' ? applyGlobalFeedLimits(feedClean, trustedGlobalAuthors) : feedClean;
   await cacheEvents(output);
   return output;
 }
 
+function applyGlobalFeedLimits(events: NostrEvent[], trustedAuthors = new Set<string>()) {
+  const trusted = events.filter((event) => trustedAuthors.has(event.pubkey));
+  const limited = limitCryptoTopicDensity(
+    limitConsecutiveAuthors(
+      events.filter((event) => !trustedAuthors.has(event.pubkey)),
+      2
+    ),
+    10
+  );
+  if (!trusted.length) return limited;
+  return dedupeEvents([...trusted, ...limited]);
+}
+
 function feedIdleMinEvents(mode: FeedMode, limit = 24) {
   if (mode === 'follow' || mode === 'custom') return Math.min(limit, 8);
   return Math.min(limit, 24);
@@ -684,7 +703,8 @@ export async function subscribeFeed(
   return pool.subscribeMap(requests, {
     label: 'main-feed-live',
     onevent(event) {
-      const [clean] = topLevelFeedEvents(filterSpam(verifiedRelayEvents([event as NostrEvent])));
+      const trustedGlobalAuthors = mode === 'global' ? new Set(options.globalAuthors?.filter((pubkey) => /^[0-9a-f]{64}$/i.test(pubkey)) ?? []) : new Set<string>();
+      const [clean] = topLevelFeedEvents(filterSpam(verifiedRelayEvents([event as NostrEvent]), new Set<string>(), trustedGlobalAuthors));
       if (clean) {
         void cacheEvents([clean]);
         onEvent(clean);

src/lib/nostr/config.ts

@@ -23,6 +23,15 @@ export const globalFeedHashtags = ['technology', 'food', 'foodstr', 'music', 'mu
 
 export const globalFeedCuratorPubkey = 'c1fc7771f5fa418fd3ac49221a18f19b42ccb7a663da8f04cbbf6c08c80d20b1';
 
+export const defaultGlobalFeedAuthors = [
+  '3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d',
+  'e83b66a8ed2d37c07d1abea6e1b000a15549c69508fa4c5875556d52b0526c2b',
+  '460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c',
+  '76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa',
+  '683211bd155c7b764e4b99ba263a151d81209be7a566a2bb1971dc1bbd3b715e',
+  'c1fc7771f5fa418fd3ac49221a18f19b42ccb7a663da8f04cbbf6c08c80d20b1'
+];
+
 export const socialInterests = [
   'art',
   'photography',

src/lib/stores/app.test.ts

@@ -40,6 +40,18 @@ describe('app store helpers', () => {
     expect(mergeEvents(items, []).map((item) => item.id)).toEqual(['a1', 'a2', 'b1', 'a3', 'a4']);
   });
 
+  it('does not de-flood explicit global authors from the global feed', () => {
+    const globalAuthor = '3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d';
+    const items = [
+      authorEvent('trusted-1', 60, globalAuthor),
+      authorEvent('trusted-2', 50, globalAuthor),
+      authorEvent('trusted-3', 40, globalAuthor),
+      authorEvent('trusted-4', 30, globalAuthor)
+    ];
+
+    expect(mergeEvents(items, []).map((item) => item.id)).toEqual(['trusted-1', 'trusted-2', 'trusted-3', 'trusted-4']);
+  });
+
   it('caps the in-memory feed after merging', () => {
     const authors = ['a'.repeat(64), 'b'.repeat(64), 'c'.repeat(64)];
     const items = Array.from({ length: 605 }, (_, index) => authorEvent(`event-${index}`, index, authors[index % authors.length]));
@@ -50,14 +62,16 @@ describe('app store helpers', () => {
     expect(merged.at(-1)?.id).toBe('event-5');
   });
 
-  it('limits global display to the default feed hashtags', () => {
+  it('limits global display to the default feed hashtags and explicit global authors', () => {
+    const globalAuthor = '3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d';
     const items = [
       { ...authorEvent('technology', 30, 'a'.repeat(64)), tags: [['t', 'technology']] },
       { ...authorEvent('food-content', 20, 'b'.repeat(64)), content: 'made lunch #foodstr' },
+      authorEvent('global-author', 15, globalAuthor),
       { ...authorEvent('other', 10, 'c'.repeat(64)), tags: [['t', 'nostr']] }
     ];
 
-    expect(displayEventsForFeedMode('global', items).map((item) => item.id)).toEqual(['technology', 'food-content']);
+    expect(displayEventsForFeedMode('global', items).map((item) => item.id)).toEqual(['technology', 'food-content', 'global-author']);
   });
 
   it('allows direct follows, friends of friends, hashtags, and keyword matches in custom display', () => {

src/lib/stores/app.ts

@@ -12,7 +12,7 @@ import {
   getCachedProfileEvents,
   getCachedProfiles
 } from '$lib/nostr/cache';
-import { defaultCustomFeedSettings, defaultGuestNip05, defaultRelays, globalFeedCuratorPubkey, globalFeedHashtags, keywordsForInterests } from '$lib/nostr/config';
+import { defaultCustomFeedSettings, defaultGlobalFeedAuthors, defaultGuestNip05, defaultRelays, globalFeedCuratorPubkey, globalFeedHashtags, keywordsForInterests } from '$lib/nostr/config';
 import { appPath } from '$lib/paths';
 import { markRelaysOffline, markRelaysOnline, syncRelayStatus } from '$lib/stores/relayStatus';
 import { insertTimelineItems, timelineCursor, uniqueFreshItems } from '$lib/timeline/window';
@@ -83,7 +83,7 @@ const maxFeedEvents = 600;
 const maxPendingNewerEvents = 600;
 const maxCachedOlderEvents = 600;
 const cachedFeedBufferLimit = maxFeedEvents + maxPendingNewerEvents + maxCachedOlderEvents;
-const olderFetchBatchLimit = 80;
+const olderFetchBatchLimit = 160;
 const olderFetchTarget = pageFeedLimit;
 const olderFetchMaxAttempts = 5;
 const olderFetchEmptyAttemptLimit = 2;
@@ -153,7 +153,7 @@ let currentSettings = defaultCustomFeedSettings;
 let currentMode: FeedMode = initialFeedMode;
 let currentHashtag = '';
 let currentSessionValue: Session | null = initialSession;
-let currentGlobalFeedAuthors: string[] = [];
+let currentGlobalFeedAuthors: string[] = defaultGlobalFeedAuthors;
 let currentNotifications: NotificationItem[] = [];
 let currentDirectMessages: DirectMessage[] = [];
 let currentNotificationSeenAt = initialSession && browser ? readLastSeen(notificationSeenStorageKey, initialSession.pubkey) : 0;
@@ -239,7 +239,10 @@ export async function bootstrap() {
   }
 
   const [cachedEvents, cachedProfiles] = await Promise.all([getCachedEvents(cachedFeedBufferLimit), getCachedProfiles()]);
-  const cachedTopLevelEvents = recentFeedEventsForMode(currentMode, cachedEventsForMode(currentMode, topLevelFeedEvents(filterSpam(cachedEvents, currentMutedPubkeys)), cachedFeedBufferLimit));
+  const cachedTopLevelEvents = recentFeedEventsForMode(
+    currentMode,
+    cachedEventsForMode(currentMode, topLevelFeedEvents(filterSpam(cachedEvents, currentMutedPubkeys, trustedFeedPubkeys(currentMode))), cachedFeedBufferLimit)
+  );
   const visibleEvents = cachedTopLevelEvents.slice(0, Math.min(initialFeedLimit, maxFeedEvents));
   cachedOlderEvents = limitFeedBuffer(cachedTopLevelEvents.slice(visibleEvents.length), maxCachedOlderEvents);
   events.set(visibleEvents);
@@ -375,12 +378,16 @@ function eventsForFeedMode(mode: FeedMode, items: NostrEvent[], follows = curren
     );
   }
   if (currentHashtag) return items.filter((event) => !eventHasImgurUrl(event));
-  return items.filter((event) => globalFeedHashtags.some((tag) => eventHasHashtag(event, tag)) && !eventHasImgurUrl(event));
+  return items.filter((event) => isGlobalFeedEvent(event));
 }
 
 async function getCachedFeedCandidates(mode: FeedMode, limit = cachedFeedBufferLimit) {
   const cachedEvents = currentHashtag ? await getCachedHashtagEvents(currentHashtag, limit) : await getCachedEvents(limit);
-  return recentFeedEventsForMode(mode, topLevelFeedEvents(filterSpam(cachedEventsForMode(mode, topLevelFeedEvents(filterSpam(cachedEvents, currentMutedPubkeys)), limit), currentMutedPubkeys)));
+  const trustedPubkeys = trustedFeedPubkeys(mode);
+  return recentFeedEventsForMode(
+    mode,
+    topLevelFeedEvents(filterSpam(cachedEventsForMode(mode, topLevelFeedEvents(filterSpam(cachedEvents, currentMutedPubkeys, trustedPubkeys)), limit), currentMutedPubkeys, trustedPubkeys))
+  );
 }
 
 async function primeCachedFeedBuffers(mode: FeedMode, visibleEvents: NostrEvent[]) {
@@ -1401,8 +1408,8 @@ function mergeFeedEvents(incoming: NostrEvent[], existing: NostrEvent[]) {
   const byId = new Map<string, NostrEvent>();
   [...existing, ...incoming].forEach((event) => byId.set(event.id, event));
   const merged = [...byId.values()].filter((event) => !currentDeletedEventIds.has(event.id)).sort((a, b) => b.created_at - a.created_at);
-  const globalClean = currentMode === 'global' ? merged.filter((event) => !eventHasImgurUrl(event)) : merged;
-  const limited = currentMode === 'global' ? limitCryptoTopicDensity(limitConsecutiveAuthors(globalClean, 2), 10) : globalClean;
+  const globalClean = currentMode === 'global' ? merged.filter((event) => currentGlobalFeedAuthors.includes(event.pubkey) || !eventHasImgurUrl(event)) : merged;
+  const limited = currentMode === 'global' ? applyGlobalFeedLimits(globalClean) : globalClean;
   return limited;
 }
 
@@ -1423,6 +1430,28 @@ function eventHasHashtag(event: NostrEvent, tag: string) {
   return new RegExp(`(^|\\s)#${normalized.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(event.content);
 }
 
+function isGlobalFeedEvent(event: NostrEvent) {
+  if (currentGlobalFeedAuthors.includes(event.pubkey)) return true;
+  return globalFeedHashtags.some((tag) => eventHasHashtag(event, tag)) && !eventHasImgurUrl(event);
+}
+
+function trustedFeedPubkeys(mode: FeedMode) {
+  return mode === 'global' && !currentHashtag ? new Set(currentGlobalFeedAuthors) : new Set<string>();
+}
+
+function applyGlobalFeedLimits(items: NostrEvent[]) {
+  const trusted = items.filter((event) => currentGlobalFeedAuthors.includes(event.pubkey));
+  const limited = limitCryptoTopicDensity(
+    limitConsecutiveAuthors(
+      items.filter((event) => !currentGlobalFeedAuthors.includes(event.pubkey)),
+      2
+    ),
+    10
+  );
+  if (!trusted.length) return limited;
+  return [...trusted, ...limited].sort((a, b) => b.created_at - a.created_at);
+}
+
 function feedKeywordHashtags(settings: CustomFeedSettings) {
   return [
     ...new Set(
@@ -2014,9 +2043,9 @@ async function resolveDefaultGlobalFeedContext() {
     mergeRelayHints(relayList.map((relay) => relay.url), 90);
     const contacts = await fetchContactListDetails(profile.pubkey, currentRelays).catch(() => ({ pubkeys: [], relayHints: [], items: [] }));
     mergeRelayHints(contacts.relayHints, 74);
-    currentGlobalFeedAuthors = mergeGlobalFeedAuthors(contacts.pubkeys, curatorContacts.pubkeys);
+    currentGlobalFeedAuthors = mergeGlobalFeedAuthors(defaultGlobalFeedAuthors, contacts.pubkeys, curatorContacts.pubkeys);
   } else {
-    currentGlobalFeedAuthors = mergeGlobalFeedAuthors(curatorContacts.pubkeys);
+    currentGlobalFeedAuthors = mergeGlobalFeedAuthors(defaultGlobalFeedAuthors, curatorContacts.pubkeys);
   }
 }