From be80062ef98012c811cdb4ebd294adf1bf62b295 Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 17 Jul 2026 04:44:30 -0500 Subject: [PATCH] fix(parsing): ingest plain-field neocities K2 Full markdown Site-scraped guides use Name:/Author: without bold markers and lack ___ separators; accept both field styles and split on ### so convert loads the full mod list. --- .../Parsing/MarkdownImportProfile.cs | 60 +- src/ModSync.Core/Parsing/MarkdownParser.cs | 5 +- src/ModSync.Tests/Fixtures/k2_full_guide.html | 5055 +++++++++++++++++ src/ModSync.Tests/Fixtures/k2_full_guide.md | 2685 +++++++++ src/ModSync.Tests/GuideIngestionTests.cs | 87 + 5 files changed, 7872 insertions(+), 20 deletions(-) create mode 100644 src/ModSync.Tests/Fixtures/k2_full_guide.html create mode 100644 src/ModSync.Tests/Fixtures/k2_full_guide.md diff --git a/src/ModSync.Core/Parsing/MarkdownImportProfile.cs b/src/ModSync.Core/Parsing/MarkdownImportProfile.cs index e32f5b39..adca8b9b 100644 --- a/src/ModSync.Core/Parsing/MarkdownImportProfile.cs +++ b/src/ModSync.Core/Parsing/MarkdownImportProfile.cs @@ -574,13 +574,30 @@ public MarkdownImportProfile Clone() public static MarkdownImportProfile CreateDefault() { - - const string defaultRawPattern = @"(?ms)^###\s*(?.+?)\s*\r?\n(?:[\s\S]*?\*\*Name:\*\*\s*(?:\[(?(?[^\]]+))\]\([^)]+\)|(?.*?))(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Author:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Description:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Masters:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Category\s*&\s*Tier:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Non-English Functionality:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Installation Method:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|\*\*Installation Instructions:\*\*\s*(?.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z)))?[\s\S]*?(?=\r?\n\s*(?:-{3,}|_{3,})|\Z)"; - - const string defaultOuterPattern = @"(?m)^###\s*.+?$[\s\S]*?(?=^___\s*$|^##\s|\Z)"; + // Shared boundary: next bold **Field:**, plain Field:, horizontal rule, or markdown heading. + // Bold alternatives are listed first so **Name:** wins when both styles appear. + const string fieldBoundary = + @"(?:\*\*[^*\r\n]{1,100}:\*\*|(?:Name|Author|Description|Masters|Category\s*&\s*Tier|Non-English Functionality|Installation Method|Installation Instructions|Install Instructions|Download Instructions|Usage Warning|Screenshots|Known Bugs|Installation Warning|Compatibility Warning|Steam Notes)\s*:|#{2,3}\s|_{3,}|-{3,})"; + + const string defaultRawPattern = + @"(?ms)^###\s*(?.+?)\s*\r?\n" + + @"(?:[\s\S]*?(?:\*\*Name:\*\*|Name:)\s*(?:\[(?(?[^\]]+))\]\([^)]+\)|(?.*?))(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Author:\*\*|Author:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Description:\*\*|Description:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Masters:\*\*|Masters:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Category\s*&\s*Tier:\*\*|Category\s*&\s*Tier:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Non-English Functionality:\*\*|Non-English Functionality:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?:\*\*Installation Method:\*\*|Installation Method:)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" + + @"(?:[\s\S]*?(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|(?:\*\*(?:Install(?:ation)?|Installation) Instructions:\*\*|(?:Install(?:ation)?|Installation) Instructions:?)\s*(?.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z)))?" + + @"[\s\S]*?(?=\r?\n\s*(?:-{3,}|_{3,})|\Z)"; + + // Also stop at the next ### so site-scraped guides without ___ separators still split per mod. + const string defaultOuterPattern = @"(?m)^###\s*.+?$[\s\S]*?(?=^###\s|^___\s*$|^##\s|\Z)"; const string defaultInstructionsBlockPattern = @""; + const string multilineFieldBody = @"(?:(?!\r?\n\s*" + fieldBoundary + @").)*"; + return new MarkdownImportProfile { Mode = RegexMode.Individual, @@ -589,22 +606,27 @@ public static MarkdownImportProfile CreateDefault() RawRegexPattern = defaultRawPattern, RawRegexOptions = RegexOptions.Multiline | RegexOptions.Singleline, HeadingPattern = @"^###\s+(?.+?)(?:\s*\[.*?\])?\s*$", - NamePattern = @"\*\*Name:\*\*\s*(?:\[(?(?[^\]]+))\]\([^)]+\)|(?[^\r\n]+))[^\r\n]*", - AuthorPattern = @"\*\*Author:\*\*\s*(?[^\r\n]+)", - DescriptionPattern = @"\*\*Description:\*\*\s*(?(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)", + NamePattern = @"(?:\*\*Name:\*\*|Name:)\s*(?:\[(?(?[^\]]+))\]\([^)]+\)|(?[^\r\n]+))[^\r\n]*", + AuthorPattern = @"(?:\*\*Author:\*\*|Author:)\s*(?[^\r\n]+)", + DescriptionPattern = @"(?:\*\*Description:\*\*|Description:)\s*(?" + multilineFieldBody + @")", ModLinkPattern = @"\[(?