improve handling invalid entities

Author: nikgraf

apps/events/src/routes/podcasts.lazy.tsx

@@ -1,4 +1,4 @@
-import { useEntities, usePublicSpaces } from '@graphprotocol/hypergraph-react';
+import { useEntities, useEntity, usePublicSpaces } from '@graphprotocol/hypergraph-react';
 import { createLazyFileRoute } from '@tanstack/react-router';
 import { Podcast, Topic } from '@/schema';
 
@@ -22,19 +22,23 @@ function RouteComponent() {
   //   }, 1000);
   // }, []);
 
-  // const { data: podcast } = useEntity(Podcast, {
-  //   id: 'f5d27d3e-3a51-452d-bac2-702574381633',
-  //   mode: 'public',
-  //   space: space,
-  //   include: {
-  //     listenOn: {},
-  //     hosts: {
-  //       avatar: {},
-  //     },
-  //     episodes: {},
-  //   },
-  // });
-  // console.log({ podcast });
+  const {
+    data: podcast,
+    invalidEntity,
+    invalidRelationEntities,
+  } = useEntity(Podcast, {
+    id: 'f5d27d3e-3a51-452d-bac2-702574381633',
+    mode: 'public',
+    space: space,
+    include: {
+      listenOn: {},
+      hosts: {
+        avatar: {},
+      },
+      episodes: {},
+    },
+  });
+  console.log({ podcast, invalidEntity, invalidRelationEntities });
 
   const { data, isLoading, isError } = useEntities(Podcast, {
     mode: 'public',

docs/docs/query-private-data.md

@@ -50,7 +50,7 @@ Please learn more about filtering in the [Filtering query results](#filtering-qu
 useEntities for private data returns:
 
 - data - a list of entities defined in your schema
-- invalidEntities - a list of entities that are in your space storage with correct type, but can't be parsed to your schema
+- invalidEntities - each entry contains the invalid raw payload (`raw`) alongside the decode `error`
 - deleted - a list of entities that are marked as deleted, we keep them around to be able to later be able to publish the deleted information to the public knowledge graph
 
 ```ts

docs/docs/query-public-data.md

@@ -86,14 +86,15 @@ Please learn more about filtering in the [Filtering query results](#filtering-qu
 useEntities for private data returns:
 
 - data - a list of entities defined in your schema
-- invalidEntities - a list of entities that are in your space storage with correct type, but can't be parsed to your schema
+- invalidEntities - each entry includes the invalid raw payload (`raw`) plus the corresponding `error` explaining why decoding failed
+- invalidRelationEntities - each entry includes the invalid raw payload (`raw`) plus the corresponding `error` explaining why decoding failed
 - isPending - a boolean indicating if the query is loading
 - isError - a boolean indicating if the query failed
 
 In addition you have access to the full response from `@tanstack/react-query`'s `useQuery` hook, which is used internally to query the public data.
 
 ```ts
-const { data, isPending, isError } = useEntities(Event, { mode: 'public' });
+const { data, invalidEntities, invalidRelationEntities, isPending, isError } = useEntities(Event, { mode: 'public' });
 ```
 
 ## Fetching a single public entity
@@ -106,7 +107,7 @@ When you only need a single entity—for example to power a detail page—you ca
 import { useEntity } from '@graphprotocol/hypergraph-react';
 import { Project } from '../schema';
 
-const { data: project, isPending, isError } = useEntity(Project, {
+const { data: project, invalidEntity, invalidRelationEntities, isPending, isError } = useEntity(Project, {
   id: '9f130661-8c3f-4db7-9bdc-3ce69631c5ef',
   space: '3f32353d-3b27-4a13-b71a-746f06e1f7db',
   mode: 'public',

packages/hypergraph-react/src/hooks/use-entity.tsx

@@ -19,5 +19,10 @@ export function useEntity<const S extends Schema.Schema.AnyNoContext>(
     return resultPublic;
   }
 
-  return resultPrivate;
+  return {
+    ...resultPublic,
+    data: resultPrivate.data,
+    invalidEntity: resultPrivate.invalidEntity,
+    invalidRelationEntities: [],
+  };
 }

packages/hypergraph-react/src/internal/use-entities-public.tsx

@@ -51,5 +51,10 @@ export const useEntitiesPublic = <S extends Schema.Schema.AnyNoContext>(type: S,
     enabled,
   });
 
-  return { ...result, data: result.data?.data || [], invalidEntities: result.data?.invalidEntities || [] };
+  return {
+    ...result,
+    data: result.data?.data || [],
+    invalidEntities: result.data?.invalidEntities || [],
+    invalidRelationEntities: result.data?.invalidRelationEntities || [],
+  };
 };

packages/hypergraph-react/src/internal/use-entity-private.tsx

@@ -18,7 +18,7 @@ export function useEntityPrivate<const S extends Schema.Schema.AnyNoContext>(
   const handle = useSubscribeToSpaceAndGetHandle({ spaceId: spaceFromParams ?? spaceFromContext, enabled });
   const prevEntityRef = useRef<{
     data: Entity.Entity<S> | undefined;
-    invalidEntity: Record<string, string | boolean | number | Date> | undefined;
+    invalidEntity: Entity.InvalidEntity | undefined;
     isPending: boolean;
     isError: boolean;
   }>({ data: undefined, invalidEntity: undefined, isPending: false, isError: false });

packages/hypergraph-react/src/internal/use-entity-public.tsx

@@ -34,5 +34,10 @@ export const useEntityPublic = <S extends Schema.Schema.AnyNoContext>(type: S, p
     enabled: enabled && !!id && !!space,
   });
 
-  return { ...result, data: result.data ?? null, invalidEntity: null };
+  return {
+    ...result,
+    data: result.data?.entity ?? null,
+    invalidEntity: result.data?.invalidEntity ?? null,
+    invalidRelationEntities: result.data?.invalidRelationEntities ?? [],
+  };
 };

packages/hypergraph/src/entity/find-many-public.ts

@@ -2,10 +2,11 @@ import { Graph } from '@graphprotocol/grc-20';
 import { Constants, type Entity, Utils } from '@graphprotocol/hypergraph';
 import * as Either from 'effect/Either';
 import * as Option from 'effect/Option';
+import type * as ParseResult from 'effect/ParseResult';
 import * as Schema from 'effect/Schema';
 import * as SchemaAST from 'effect/SchemaAST';
 import { request } from 'graphql-request';
-import type { RelationsListWithNodes } from '../utils/convert-relations.js';
+import type { InvalidRelationEntity, RelationsListWithNodes } from '../utils/convert-relations.js';
 import type { RelationTypeIdInfo } from '../utils/get-relation-type-ids.js';
 import { buildRelationsSelection } from '../utils/relation-query-helpers.js';
 import type { SpaceSelection } from './internal/space-selection.js';
@@ -110,6 +111,13 @@ type ValuesList = {
   point: string;
 }[];
 
+type RawEntity = Record<string, string | boolean | number | unknown[] | Date>;
+
+export type InvalidEntity = {
+  raw: RawEntity;
+  error: ParseResult.ParseError;
+};
+
 export type EntityQueryResult = {
   entities: ({
     id: string;
@@ -134,10 +142,11 @@ export const parseResult = <S extends Schema.Schema.AnyNoContext>(
   const schemaWithId = Utils.addIdSchemaField(type);
   const decode = Schema.decodeUnknownEither(schemaWithId);
   const data: (Entity.Entity<S> & { backlinksTotalCountsTypeId1?: number })[] = [];
-  const invalidEntities: Record<string, unknown>[] = [];
+  const invalidEntities: InvalidEntity[] = [];
+  const invalidRelationEntities: InvalidRelationEntity[] = [];
 
   for (const queryEntity of queryData.entities) {
-    let rawEntity: Record<string, string | boolean | number | unknown[] | Date> = {
+    let rawEntity: RawEntity = {
       id: queryEntity.id,
     };
 
@@ -162,11 +171,19 @@ export const parseResult = <S extends Schema.Schema.AnyNoContext>(
       }
     }
 
+    const { rawEntity: relationEntities, invalidRelations } = Utils.convertRelations(
+      queryEntity,
+      ast,
+      relationInfoLevel1,
+    );
     // @ts-expect-error
     rawEntity = {
       ...rawEntity,
-      ...Utils.convertRelations(queryEntity, ast, relationInfoLevel1),
+      ...relationEntities,
     };
+    if (invalidRelations.length > 0) {
+      invalidRelationEntities.push(...invalidRelations);
+    }
 
     const decodeResult = decode({
       ...rawEntity,
@@ -181,10 +198,10 @@ export const parseResult = <S extends Schema.Schema.AnyNoContext>(
         backlinksTotalCountsTypeId1: queryEntity.backlinksTotalCountsTypeId1?.totalCount,
       });
     } else {
-      invalidEntities.push(rawEntity);
+      invalidEntities.push({ raw: rawEntity, error: decodeResult.left });
     }
   }
-  return { data, invalidEntities };
+  return { data, invalidEntities, invalidRelationEntities };
 };
 
 export const findManyPublic = async <S extends Schema.Schema.AnyNoContext>(
@@ -271,6 +288,12 @@ export const findManyPublic = async <S extends Schema.Schema.AnyNoContext>(
 
   const result = await request<EntityQueryResult>(`${Graph.TESTNET_API_ORIGIN}/graphql`, queryDocument, queryVariables);
 
-  const { data, invalidEntities } = parseResult(result, type, relationTypeIds);
-  return { data, invalidEntities };
+  const { data, invalidEntities, invalidRelationEntities } = parseResult(result, type, relationTypeIds);
+  if (invalidEntities.length > 0) {
+    console.warn('Entities where decoding failed were dropped', invalidEntities);
+  }
+  if (invalidRelationEntities.length > 0) {
+    console.warn('Relation entities where decoding failed were dropped', invalidRelationEntities);
+  }
+  return { data, invalidEntities, invalidRelationEntities };
 };

packages/hypergraph/src/entity/find-one-public.ts

@@ -49,7 +49,11 @@ const parseResult = <S extends Schema.Schema.AnyNoContext>(
   relationInfoLevel1: RelationTypeIdInfo[],
 ) => {
   if (!queryData.entity) {
-    return null;
+    return {
+      entity: null,
+      invalidEntity: null,
+      invalidRelationEntities: [],
+    };
   }
 
   const schemaWithId = Utils.addIdSchemaField(type);
@@ -80,25 +84,39 @@ const parseResult = <S extends Schema.Schema.AnyNoContext>(
     }
   }
 
+  const { rawEntity: relationEntities, invalidRelations } = Utils.convertRelations(
+    queryEntity,
+    ast,
+    relationInfoLevel1,
+  );
   // @ts-expect-error
   rawEntity = {
     ...rawEntity,
-    ...Utils.convertRelations(queryEntity, ast, relationInfoLevel1),
+    ...relationEntities,
   };
+  if (invalidRelations.length > 0) {
+    console.warn('Relation entities where decoding failed were dropped', invalidRelations);
+  }
 
   const decodeResult = decode({
     ...rawEntity,
     __deleted: false,
   });
 
   if (Either.isRight(decodeResult)) {
-    return { ...decodeResult.right } as Entity.Entity<S>;
+    return {
+      entity: { ...decodeResult.right } as Entity.Entity<S>,
+      invalidEntity: null,
+      invalidRelationEntities: invalidRelations,
+    };
   }
 
-  // if (process.env.NODE_ENV !== 'production') {
-  console.warn('Invalid entity', rawEntity);
-  // }
-  throw new Error('Invalid entity');
+  console.warn('Entity decoding failed', { raw: rawEntity, error: decodeResult.left });
+  return {
+    entity: null,
+    invalidEntity: { raw: rawEntity, error: decodeResult.left },
+    invalidRelationEntities: invalidRelations,
+  };
 };
 
 export const findOnePublic = async <S extends Schema.Schema.AnyNoContext>(type: S, params: FindOnePublicParams<S>) => {

packages/hypergraph/src/utils/convert-relations.ts

@@ -1,6 +1,8 @@
 import { Constants, Utils } from '@graphprotocol/hypergraph';
+import * as Either from 'effect/Either';
 import * as Option from 'effect/Option';
-import type * as Schema from 'effect/Schema';
+import type * as ParseResult from 'effect/ParseResult';
+import * as Schema from 'effect/Schema';
 import * as SchemaAST from 'effect/SchemaAST';
 import { convertPropertyValue } from './convert-property-value.js';
 import type { RelationTypeIdInfo } from './get-relation-type-ids.js';
@@ -46,12 +48,30 @@ type RecursiveQueryEntity = {
 type RawEntityValue = string | boolean | number | unknown[] | Date | { id: string };
 type RawEntity = Record<string, RawEntityValue>;
 type NestedRawEntity = RawEntity & { _relation: { id: string } & Record<string, RawEntityValue> };
+
+export type InvalidRelationEntity = {
+  parentEntityId: string;
+  propertyName: string;
+  propertyTypeId: string;
+  relationId: string;
+  relationTypeId: string;
+  toEntityId: string;
+  raw: NestedRawEntity;
+  error: ParseResult.ParseError;
+};
+
+type ConvertRelationsResult = {
+  rawEntity: RawEntity;
+  invalidRelations: InvalidRelationEntity[];
+};
+
 export const convertRelations = <_S extends Schema.Schema.AnyNoContext>(
   queryEntity: RecursiveQueryEntity,
   ast: SchemaAST.TypeLiteral,
   relationInfo: RelationTypeIdInfo[] = [],
-) => {
+): ConvertRelationsResult => {
   const rawEntity: RawEntity = {};
+  const invalidRelations: InvalidRelationEntity[] = [];
 
   for (const prop of ast.propertySignatures) {
     const result = SchemaAST.getAnnotation<string>(Constants.PropertyIdSymbol)(prop.type);
@@ -93,6 +113,9 @@ export const convertRelations = <_S extends Schema.Schema.AnyNoContext>(
         }
       }
 
+      const relationSchema = Schema.make(relationTransformation);
+      const decodeRelation = Schema.decodeUnknownEither(relationSchema);
+
       if (allRelationsWithTheCorrectPropertyTypeId) {
         for (const relationEntry of allRelationsWithTheCorrectPropertyTypeId) {
           let nestedRawEntity: NestedRawEntity = {
@@ -102,16 +125,17 @@ export const convertRelations = <_S extends Schema.Schema.AnyNoContext>(
             },
           };
 
-          const relationsForRawNestedEntity = convertRelations(
+          const childConversion = convertRelations(
             relationEntry.toEntity,
             relationTransformation,
             relationMetadata?.children ?? [],
           );
 
           nestedRawEntity = {
             ...nestedRawEntity,
-            ...relationsForRawNestedEntity,
+            ...childConversion.rawEntity,
           };
+          invalidRelations.push(...childConversion.invalidRelations);
 
           for (const nestedProp of relationTransformation.propertySignatures) {
             const propType =
@@ -156,8 +180,21 @@ export const convertRelations = <_S extends Schema.Schema.AnyNoContext>(
             }
           }
 
-          // TODO: in the end every entry should be validated using the Schema?!?
-          rawEntity[String(prop.name)] = [...(rawEntity[String(prop.name)] as unknown[]), nestedRawEntity];
+          const decodedRelation = decodeRelation(nestedRawEntity);
+          if (Either.isRight(decodedRelation)) {
+            rawEntity[String(prop.name)] = [...(rawEntity[String(prop.name)] as unknown[]), nestedRawEntity];
+          } else {
+            invalidRelations.push({
+              parentEntityId: queryEntity.id,
+              propertyName: String(prop.name),
+              propertyTypeId: result.value,
+              relationId: relationEntry.id,
+              relationTypeId: relationEntry.typeId,
+              toEntityId: relationEntry.toEntity.id,
+              raw: nestedRawEntity,
+              error: decodedRelation.left,
+            });
+          }
         }
       }
 
@@ -167,5 +204,5 @@ export const convertRelations = <_S extends Schema.Schema.AnyNoContext>(
     }
   }
 
-  return rawEntity;
+  return { rawEntity, invalidRelations };
 };