Skip to content

fix(parsing): K2 Full neocities plain-field markdown ingest #183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
oldrepublicwizard merged 1 commit into from
Jul 17, 2026
+7,872 −20
Merged

fix(parsing): K2 Full neocities plain-field markdown ingest #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 41 additions & 19 deletions src/ModSync.Core/Parsing/MarkdownImportProfile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,13 +574,30 @@ public MarkdownImportProfile Clone()

public static MarkdownImportProfile CreateDefault()
{

const string defaultRawPattern = @"(?ms)^###\s*(?<heading>.+?)\s*\r?\n(?:[\s\S]*?\*\*Name:\*\*\s*(?:\[(?<name>(?<name_link>[^\]]+))\]\([^)]+\)|(?<name_plain>.*?))(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Author:\*\*\s*(?<author>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Description:\*\*\s*(?<description>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Masters:\*\*\s*(?<masters>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Category\s*&\s*Tier:\*\*\s*(?<category_tier>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Non-English Functionality:\*\*\s*(?<non_english>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?\*\*Installation Method:\*\*\s*(?<installation_method>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z))?(?:[\s\S]*?(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?<installation_instructions>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|\*\*Installation Instructions:\*\*\s*(?<installation_instructions>.*?)(?=\r?\n\s*\*\*[^:\n]{1,100}:\*\*|\r?\n\s*(?:-{3,}|_{3,})|\Z)))?[\s\S]*?(?=\r?\n\s*(?:-{3,}|_{3,})|\Z)";

const string defaultOuterPattern = @"(?m)^###\s*.+?$[\s\S]*?(?=^___\s*$|^##\s|\Z)";
// Shared boundary: next bold **Field:**, plain Field:, horizontal rule, or markdown heading.
// Bold alternatives are listed first so **Name:** wins when both styles appear.
const string fieldBoundary =
@"(?:\*\*[^*\r\n]{1,100}:\*\*|(?:Name|Author|Description|Masters|Category\s*&\s*Tier|Non-English Functionality|Installation Method|Installation Instructions|Install Instructions|Download Instructions|Usage Warning|Screenshots|Known Bugs|Installation Warning|Compatibility Warning|Steam Notes)\s*:|#{2,3}\s|_{3,}|-{3,})";

const string defaultRawPattern =
@"(?ms)^###\s*(?<heading>.+?)\s*\r?\n" +
@"(?:[\s\S]*?(?:\*\*Name:\*\*|Name:)\s*(?:\[(?<name>(?<name_link>[^\]]+))\]\([^)]+\)|(?<name_plain>.*?))(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Author:\*\*|Author:)\s*(?<author>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Description:\*\*|Description:)\s*(?<description>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Masters:\*\*|Masters:)\s*(?<masters>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Category\s*&\s*Tier:\*\*|Category\s*&\s*Tier:)\s*(?<category_tier>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Non-English Functionality:\*\*|Non-English Functionality:)\s*(?<non_english>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?:\*\*Installation Method:\*\*|Installation Method:)\s*(?<installation_method>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z))?" +
@"(?:[\s\S]*?(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?<installation_instructions>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|(?:\*\*(?:Install(?:ation)?|Installation) Instructions:\*\*|(?:Install(?:ation)?|Installation) Instructions:?)\s*(?<installation_instructions>.*?)(?=\r?\n\s*" + fieldBoundary + @"|\Z)))?" +
@"[\s\S]*?(?=\r?\n\s*(?:-{3,}|_{3,})|\Z)";

// Also stop at the next ### so site-scraped guides without ___ separators still split per mod.
const string defaultOuterPattern = @"(?m)^###\s*.+?$[\s\S]*?(?=^###\s|^___\s*$|^##\s|\Z)";

const string defaultInstructionsBlockPattern = @"<!--<<ModSync>>\s*(?<instructions>[\s\S]*?)-->";

const string multilineFieldBody = @"(?:(?!\r?\n\s*" + fieldBoundary + @").)*";

return new MarkdownImportProfile
{
Mode = RegexMode.Individual,
Expand All @@ -589,22 +606,27 @@ public static MarkdownImportProfile CreateDefault()
RawRegexPattern = defaultRawPattern,
RawRegexOptions = RegexOptions.Multiline | RegexOptions.Singleline,
HeadingPattern = @"^###\s+(?<heading>.+?)(?:\s*\[.*?\])?\s*$",
NamePattern = @"\*\*Name:\*\*\s*(?:\[(?<name>(?<name_link>[^\]]+))\]\([^)]+\)|(?<name_plain>[^\r\n]+))[^\r\n]*",
AuthorPattern = @"\*\*Author:\*\*\s*(?<author>[^\r\n]+)",
DescriptionPattern = @"\*\*Description:\*\*\s*(?<description>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
NamePattern = @"(?:\*\*Name:\*\*|Name:)\s*(?:\[(?<name>(?<name_link>[^\]]+))\]\([^)]+\)|(?<name_plain>[^\r\n]+))[^\r\n]*",
AuthorPattern = @"(?:\*\*Author:\*\*|Author:)\s*(?<author>[^\r\n]+)",
DescriptionPattern = @"(?:\*\*Description:\*\*|Description:)\s*(?<description>" + multilineFieldBody + @")",
ModLinkPattern = @"\[(?<label>[^]]+)\]\((?<link>[^)]+)\)",
CategoryTierPattern = @"\*\*Category\s*&\s*Tier:\*\*\s*(?<category>[^/\r\n]+)/\s*(?<tier>[^\r\n]+)",
InstallationMethodPattern = @"\*\*Installation Method:\*\*\s*(?<method>[^\r\n]+)",
DownloadInstructionsPattern = @"\*\*Download Instructions:\*\*\s*(?<download>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
InstallationInstructionsPattern = @"(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?<directions>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|(?:\*\*(?:Install(?:ation)?|Installation) Instructions:\*\*\s*(?<directions>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)))",
UsageWarningPattern = @"\*\*Usage Warning:\*\*\s*(?<warning>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
ScreenshotsPattern = @"\*\*Screenshots:\*\*\s*(?<screenshots>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
KnownBugsPattern = @"(?::::warning\s*\r?\n\s*Known Bugs\s*\r?\n:\s*(?<bugs>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::|\*\*Known Bugs:\*\*\s*(?<bugs>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*))",
InstallationWarningPattern = @"\*\*Installation Warning:\*\*\s*(?<installwarning>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
CompatibilityWarningPattern = @"\*\*Compatibility Warning:\*\*\s*(?<compatwarning>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
SteamNotesPattern = @"\*\*Steam Notes:\*\*\s*(?<steamnotes>(?:(?!\r?\n\s*(?:\*\*\w+[^:]*:\*\*|_{3,}|-{3,}|##)).)*)",
NonEnglishPattern = @"\*\*Non-English Functionality:\*\*\s*(?<value>[^\r\n]+)",
DependenciesPattern = @"\*\*Masters:\*\*\s*(?<masters>[^\r\n]+)",
CategoryTierPattern = @"(?:\*\*Category\s*&\s*Tier:\*\*|Category\s*&\s*Tier:)\s*(?<category>[^/\r\n]+)/\s*(?<tier>[^\r\n]+)",
InstallationMethodPattern = @"(?:\*\*Installation Method:\*\*|Installation Method:)\s*(?<method>[^\r\n]+)",
DownloadInstructionsPattern = @"(?:\*\*Download Instructions:\*\*|Download Instructions:?)\s*(?<download>" + multilineFieldBody + @")",
InstallationInstructionsPattern =
@"(?::::note\s*\r?\n\s*Installation Instructions\s*\r?\n:\s*(?<directions>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::" +
@"|(?:\*\*(?:Install(?:ation)?|Installation) Instructions:\*\*\s*(?<directions>" + multilineFieldBody + @"))" +
@"|(?m)^(?:Install(?:ation)?|Installation) Instructions:?\s*(?:\r?\n)+(?<directions>" + multilineFieldBody + @"))",
UsageWarningPattern = @"(?:\*\*Usage Warning:\*\*|Usage Warning:?)\s*(?<warning>" + multilineFieldBody + @")",
ScreenshotsPattern = @"(?:\*\*Screenshots:\*\*|Screenshots:?)\s*(?<screenshots>" + multilineFieldBody + @")",
KnownBugsPattern =
@"(?::::warning\s*\r?\n\s*Known Bugs\s*\r?\n:\s*(?<bugs>(?:(?!\r?\n\s*:::).)*?)\r?\n\s*:::" +
@"|(?:\*\*Known Bugs:\*\*|Known Bugs:?)\s*(?<bugs>" + multilineFieldBody + @"))",
InstallationWarningPattern = @"(?:\*\*Installation Warning:\*\*|Installation Warning:?)\s*(?<installwarning>" + multilineFieldBody + @")",
CompatibilityWarningPattern = @"(?:\*\*Compatibility Warning:\*\*|Compatibility Warning:?)\s*(?<compatwarning>" + multilineFieldBody + @")",
SteamNotesPattern = @"(?:\*\*Steam Notes:\*\*|Steam Notes:?)\s*(?<steamnotes>" + multilineFieldBody + @")",
NonEnglishPattern = @"(?:\*\*Non-English Functionality:\*\*|Non-English Functionality:)\s*(?<value>[^\r\n]+)",
DependenciesPattern = @"(?:\*\*Masters:\*\*|Masters:)\s*(?<masters>[^\r\n]+)",
DependenciesSeparatorPattern = @"[,;+&]",
RestrictionsPattern = string.Empty,
OptionPattern = string.Empty,
Expand Down
5 changes: 4 additions & 1 deletion src/ModSync.Core/Parsing/MarkdownParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,10 @@ private ModComponent ParseComponentFromText([NotNull] string componentText, out
if (nameFieldMatch.Success)
{

component.NameFieldContent = nameFieldMatch.Value.Replace("**Name:**", "").Trim();
// Strip bold or plain Name: label from captured field text.
component.NameFieldContent = Regex
.Replace(nameFieldMatch.Value, @"^\s*(?:\*\*Name:\*\*|Name:)\s*", string.Empty, RegexOptions.None, TimeSpan.FromSeconds(1))
.Trim();
_logVerbose($" Captured full Name field content: '{component.NameFieldContent.Substring(0, Math.Min(100, component.NameFieldContent.Length))}...'");
}
}
Expand Down
Loading
Loading