Skip to content

Commit d7eba22

Browse files
BellCubeDevBellCubeDev
authored
Merge pull request #39 from BellCubeDev/development
✨ Small Feature Additions
2 parents 77fade6 + fb0b753 commit d7eba22

8 files changed

Lines changed: 91 additions & 16 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"prepare": "husky; pnpm run prepare:wiki-repo; pnpm run prepare:schemas; pnpm run prepare:parsoid-service",
1212
"prepare:parsoid-service": "cd ./node_modules/parsoid-service; composer install",
1313
"prepare:wiki-repo": "If (-not (Test-Path -Path './cache/github-wiki/')) { git clone https://github.com/BellCubeDev/papyrus-index.wiki.git ./cache/github-wiki/ } else { git -C ./cache/github-wiki/ pull }",
14-
"prepare:schemas": "typescript-json-schema './src/papyrus/data-structures/pure/scriptSource.ts' PapyrusScriptSourceMetadata --required --noExtraProps --strictNullChecks --skipLibCheck -o './data/SourceMetadata.schema.json' --include './src/papyrus/data-structures/pure/**'; typescript-json-schema './src/papyrus/data-structures/ignore-yaml.ts' IgnoreYaml --required --noExtraProps --strictNullChecks --skipLibCheck -o './data/IgnoreYaml.schema.json' --include './src/papyrus/data-structures/ignore-yaml.ts'",
14+
"prepare:schemas": "typescript-json-schema './src/papyrus/data-structures/pure/scriptSource.ts' PapyrusScriptSourceMetadataForSchema --required --noExtraProps --strictNullChecks --skipLibCheck -o './data/SourceMetadata.schema.json' --include './src/papyrus/data-structures/pure/**'; typescript-json-schema './src/papyrus/data-structures/ignore-yaml.ts' IgnoreYaml --required --noExtraProps --strictNullChecks --skipLibCheck -o './data/IgnoreYaml.schema.json' --include './src/papyrus/data-structures/ignore-yaml.ts'",
1515
"download-mods": "tsx --env-file-if-exists=./.env ./src/papyrus/scraping/download/DownloadNeededMods.ts",
1616
"parse:reset": "rm -f ./public/raw.json",
1717
"parse:raw": "tsx ./src/papyrus/parsing/parse-or-load-all.ts",

‎src/app/[game]/source/[source]/page.tsx‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ import type { Metadata } from "next";
33
import React from "react";
44
import { UnreachableError } from "../../../../UnreachableError";
55
import type { PapyrusGame } from "../../../../papyrus/data-structures/pure/game";
6-
import { PapyrusSourceType, type PapyrusScriptSourceMetadata, type PapyrusScriptSourceMetadataVanilla, type PapyrusScriptSourceMetadataWithGitHub, type PapyrusScriptSourceMetadataXSE } from "../../../../papyrus/data-structures/pure/scriptSource";
6+
import { PapyrusSourceType, type PapyrusScriptSourceMetadata, type PapyrusScriptSourceMetadataVanilla } from "../../../../papyrus/data-structures/pure/scriptSource";
77
import { getGameName } from "../../../../utils/getGameName";
88
import { SourceName } from "../../../components/papyrus/SourceName";
99
import { getGameAndSourceFromParams, type SourceRouteParams } from "./getGameAndSourceFromParams";
1010
import { AllScripts } from "../../../../papyrus/parsing/parse-or-load-all";
1111
import { toLowerCase } from "../../../../utils/toLowerCase";
12+
import { AllScriptsIndexed } from "../../../../papyrus/indexing/index-all";
13+
import { PapyrusScriptReference } from "../../../components/papyrus/script/PapyrusScriptReference";
1214

1315
export function generateStaticParams(): SourceRouteParams[] {
1416
const params = [];
@@ -30,22 +32,33 @@ export async function generateMetadata({params}: {params: Promise<SourceRoutePar
3032

3133
export default async function SourcePage({params}: {readonly params: Promise<SourceRouteParams>}) {
3234
const {game, source} = getGameAndSourceFromParams(await params);
35+
const gameData = AllScriptsIndexed[game];
3336

34-
return <>
37+
return <main>
3538
<SourcePageSourceData game={game} sourceData={source} />
36-
37-
</>;
38-
39+
<h2>Scripts In This Source</h2>
40+
<ul>
41+
{Object.entries(gameData.scriptSources[source.sourceIdentifier]!.scripts).map(([scriptIdentifier, script]) =>
42+
<li key={scriptIdentifier}>
43+
<PapyrusScriptReference
44+
game={game}
45+
script={script}
46+
inTooltip={false}
47+
/>
48+
</li>
49+
)}
50+
</ul>
51+
</main>;
3952
}
4053

4154
function SourcePageVanillaGameData<TGame extends PapyrusGame>({game, sourceData}: {readonly game: TGame, readonly sourceData: PapyrusScriptSourceMetadataVanilla<TGame>}) {
4255
return <>
43-
<h1>{getGameName(game)} - <SourceName source={sourceData} long /></h1>
56+
<h1>{getGameName(game)} (the vanilla game)</h1>
4457
<p>Scripts included in the vanilla game. Users will not need to download anything.</p>
4558
</>;
4659
}
4760

48-
function SourcePageScriptExtenderData<TGame extends PapyrusGame>({game, sourceData}: {readonly game: TGame, readonly sourceData: PapyrusScriptSourceMetadataXSE<TGame>}) {
61+
function SourcePageScriptExtenderData<TGame extends PapyrusGame>({game, sourceData}: {readonly game: TGame, readonly sourceData: PapyrusScriptSourceMetadata<TGame> & {type: PapyrusSourceType.xSE}}) {
4962
return <>
5063
<h1>{getGameName(game)} - <SourceName source={sourceData} long /></h1>
5164
<p>
@@ -60,7 +73,7 @@ function SourcePageScriptExtenderData<TGame extends PapyrusGame>({game, sourceDa
6073
</>;
6174
}
6275

63-
function SourcePageExternalSourceData<TGame extends PapyrusGame>({game, sourceData}: {readonly game: TGame, readonly sourceData: PapyrusScriptSourceMetadataWithGitHub<TGame>}) {
76+
function SourcePageExternalSourceData<TGame extends PapyrusGame>({game, sourceData}: {readonly game: TGame, readonly sourceData: PapyrusScriptSourceMetadata<TGame> & {type: Exclude<PapyrusSourceType, PapyrusSourceType.xSE|PapyrusSourceType.Vanilla>}}) {
6477
return <>
6578
<h1>{getGameName(game)} - <SourceName source={sourceData} long /></h1>
6679
<p>This source is a mod and will need to be downloaded & installed separately by users. {sourceData.type !== PapyrusSourceType.PapyrusLib ? null : <>

‎src/app/components/papyrus/SourceName.tsx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import type { PapyrusGame } from "../../../papyrus/data-structures/pure/game";
2-
import { PapyrusSourceType, type PapyrusScriptSourceMetadataExternal, type PapyrusScriptSourceMetadataVanilla } from "../../../papyrus/data-structures/pure/scriptSource";
2+
import { PapyrusSourceType, type PapyrusScriptSourceMetadata } from "../../../papyrus/data-structures/pure/scriptSource";
3+
import { getGameName } from "../../../utils/getGameName";
34

4-
export function SourceName({source, long}: {source: PapyrusScriptSourceMetadataVanilla<PapyrusGame> | PapyrusScriptSourceMetadataExternal<PapyrusGame>, long?: boolean}): string {
5+
export function SourceName({source, long}: {source: PapyrusScriptSourceMetadata<PapyrusGame>, long?: boolean}): string {
56
switch (source.type) {
67
case PapyrusSourceType.Vanilla:
7-
return long ? 'The Vanilla Game' : 'Vanilla';
8+
return long ? `vanilla ${getGameName(source.game)}` : 'Vanilla';
89
default:
910
return long ? source.nameProper : source.nameShort;
1011
}

‎src/mediawiki/data-extraction/getMediaWikiFunctionData.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import { appendToStepSummarySection, StepSummarySection } from "../../utils/step
1010
import { getBestName, getBestNameVariant } from "../../utils/getBestName";
1111
import { extractLinearWikiPageData } from "./parsoidToPageData";
1212
import { memoizeDevServerConst } from "../../utils/memoizeDevServerConst";
13+
import { AllScriptsIndexed } from "../../papyrus/indexing/index-all";
14+
import { AllSourcesCombined } from "../../papyrus/data-structures/indexing/game";
1315

1416
export type PotentialFunction<TGame extends PapyrusGame> = PapyrusScriptFunctionIndexedAggregate<TGame>| PapyrusScriptFunction<TGame> | PapyrusScriptFunctionIndexed<TGame>;
1517

@@ -71,8 +73,61 @@ async function getMediaWikiFunctionDataInternal<TGame extends PapyrusGame, TFunc
7173
const document = await getWikiPageHTMLDocument(wiki, pageName);
7274
if (!document) return null;
7375

76+
// example redirect: <link rel="mw:PageProp/redirect" href="./GetReference_-_ReferenceAlias"
77+
// data-parsoid='{"src":"#REDIRECT ","a":{"href":"./GetReference_-_ReferenceAlias"},"sa":{"href":"GetReference - ReferenceAlias"},"dsr":[0,43,null,null]}' />
78+
const redirectElement = document.documentElement.querySelector('link[rel="mw:PageProp/redirect"');
79+
if (redirectElement) {
80+
const redirectHref = redirectElement.getAttribute('href');
81+
if (!redirectHref) {
82+
console.warn(`[MediaWiki Scraping - getWikiDataFunctionPage()] redirect for function page "${pageName}" on wiki "${wiki.wikiName}" (${document.location.href}) has no href attribute!`);
83+
appendToStepSummarySection(`
84+
### Function Page w/ Invalid Redirect (No href attribute)
85+
- **Wiki**: [${wiki.wikiName}](${wiki.wikiBaseUrl})
86+
- **Wiki Page:** [${pageName}](${document.location.href})
87+
- **Function:** ${scriptName}.${functionName}
88+
`.trim(), StepSummarySection.MediaWikiFormattingWarnings);
89+
return null;
90+
}
91+
92+
const redirectedUrl = new URL(redirectHref, document.location.href);
93+
94+
const match = redirectedUrl.pathname.match(/^\/wiki\/(?<redirectedFunctionName>\w+)_-_(?<redirectedScriptName>\w+)$/u);
95+
const {redirectedFunctionName, redirectedScriptName} = match?.groups ?? {};
96+
97+
if (!redirectedFunctionName || !redirectedScriptName) {
98+
console.warn(`[MediaWiki Scraping - getWikiDataFunctionPage()] redirect for function page "${pageName}" on wiki "${wiki.wikiName}" (${document.location.href}) redirects to a page that does not fit the expected function page name format!`, {redirectedFunctionName, redirectedScriptName, redirectedUrl, match});
99+
appendToStepSummarySection(`
100+
### Function Page w/ Invalid Redirect (Invalid Page Name Format)
101+
- **Wiki**: [${wiki.wikiName}](${wiki.wikiBaseUrl})
102+
- **Wiki Page:** [${pageName}](${document.location.href})
103+
- **Redirected To:** [${redirectHref}](${redirectedUrl})
104+
- **Function:** ${scriptName}.${functionName}
105+
- **Redirected Function:** ${redirectedScriptName}.${redirectedFunctionName}
106+
`.trim(), StepSummarySection.MediaWikiFormattingWarnings);
107+
return null;
108+
}
109+
110+
const redirectedFunction = AllScriptsIndexed[game].scripts[toLowerCase(redirectedScriptName)]?.[AllSourcesCombined].functions[toLowerCase(redirectedFunctionName)];
111+
if (!redirectedFunction) {
112+
console.warn(`[MediaWiki Scraping - getWikiDataFunctionPage()] redirect for function page "${pageName}" on wiki "${wiki.wikiName}" (${document.location.href}) redirects to a function that does not exist in the index! Redirected URL is ${redirectHref}, and the function should be ${redirectedScriptName}.${redirectedFunctionName}`, {redirectedFunctionName, redirectedScriptName, redirectedUrl, match});
113+
appendToStepSummarySection(`
114+
### Function Page w/ Invalid Redirect (Function Not Found in Index)
115+
- **Wiki**: [${wiki.wikiName}](${wiki.wikiBaseUrl})
116+
- **Wiki Page:** [${pageName}](${document.location.href})
117+
- **Redirected To:** [${redirectHref}](${redirectedUrl})
118+
- **Function:** ${scriptName}.${functionName}
119+
- **Redirected Function:** ${redirectedScriptName}.${redirectedFunctionName}
120+
`.trim(), StepSummarySection.MediaWikiFormattingWarnings);
121+
return null;
122+
}
123+
124+
return getMediaWikiFunctionData(game, redirectedFunction, redirectedScriptName);
125+
}
126+
74127
const pageData = extractLinearWikiPageData(document);
75128

129+
//console.log(pageData);
130+
76131
// TODO: Better support the formatting on display in the Skyrim CK wiki's ColorComponent script,
77132
// especially the parameters section. That, or contribute to the wiki and standardize it.
78133
// e.g. https://ck.uesp.net/wiki/GetAlpha_-_ColorComponent

‎src/mediawiki/data-extraction/parsoidToPageData.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export function extractLinearWikiPageData(document: Document): LinearMediaWikiPa
4343
let isIncomplete = false;
4444
let minimumXSEVersion: string | null = null;
4545

46-
// data-mw='{"parts":[{"template":{"target":{"wt":"Template:Incomplete Article","href":"./Template:Incomplete_Article"},"params":{},"i":0}}]}'
46+
// example: data-mw='{"parts":[{"template":{"target":{"wt":"Template:Incomplete Article","href":"./Template:Incomplete_Article"},"params":{},"i":0}}]}'
4747
const transclusions = document.body.querySelectorAll('[typeof="mw:Transclusion"]');
4848
for (const el of transclusions) {
4949
removeElement(el); // so we don't include this data when we don't mean to

‎src/papyrus/data-structures/pure/scriptSource.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export interface PapyrusScriptSourceScriptData<TGame extends PapyrusGame> {
5858
scripts: Record<Lowercase<string>, PapyrusScript<TGame>>;
5959
}
6060

61-
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- may be used later
61+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
6262
export interface PapyrusScriptSourceMetadataBase<TGame extends PapyrusGame> {
6363
type: PapyrusSourceType;
6464

@@ -69,6 +69,10 @@ export interface PapyrusScriptSourceMetadataBase<TGame extends PapyrusGame> {
6969
allowedScripts?: string[];
7070
}
7171

72+
export interface PapyrusScriptSourceMetadataDynamicAdditions<TGame extends PapyrusGame> {
73+
game: TGame;
74+
}
75+
7276
export interface PapyrusScriptSourceMetadataVanilla<TGame extends PapyrusGame> extends PapyrusScriptSourceMetadataBase<TGame> {
7377
type: PapyrusSourceType.Vanilla;
7478
}
@@ -115,5 +119,6 @@ export interface PapyrusScriptSourceMetadataWithGitHub<TGame extends PapyrusGame
115119
}
116120

117121
export type PapyrusScriptSourceMetadataExternal<TGame extends PapyrusGame> = PapyrusScriptSourceMetadataXSE<TGame> | PapyrusScriptSourceMetadataWithGitHub<TGame>;
118-
export type PapyrusScriptSourceMetadata<TGame extends PapyrusGame> = PapyrusScriptSourceMetadataVanilla<TGame> | PapyrusScriptSourceMetadataExternal<TGame>;
122+
export type PapyrusScriptSourceMetadataForSchema<TGame extends PapyrusGame> =PapyrusScriptSourceMetadataVanilla<TGame> | PapyrusScriptSourceMetadataExternal<TGame>;
123+
export type PapyrusScriptSourceMetadata<TGame extends PapyrusGame> = PapyrusScriptSourceMetadataDynamicAdditions<TGame> & PapyrusScriptSourceMetadataForSchema<TGame>;
119124
export type PapyrusScriptSource<TGame extends PapyrusGame> = PapyrusScriptSourceScriptData<TGame> & PapyrusScriptSourceMetadata<TGame>;

‎src/papyrus/indexing/index-all.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ function getAllScriptsIndexed()
1111
{
1212
return Object.fromEntries(Object.values(PapyrusGame).map((game) =>
1313
[game, getAllScriptsIndexedForGame(game)]
14-
)) as {[K in PapyrusGame]: ReturnType<typeof indexGame>};
14+
)) as {[K in PapyrusGame]: ReturnType<typeof indexGame<K>>};
1515
}
1616

1717
function getAllScriptsIndexedForGame<TGame extends PapyrusGame>(game: TGame) {

‎src/papyrus/parsing/parse-all-for-game.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ export async function parseAllScriptsForGame<TGame extends PapyrusGame>(game: TG
131131
return {
132132
...sourceMetadata,
133133
sourceIdentifier: toLowerCase(sourceIdentifier.name),
134+
game,
134135
scripts
135136
};
136137
}))).filter((x): x is Exclude<typeof x, null> => x !== null);

0 commit comments

Comments
 (0)