diff --git a/Tests/Images/Ja-Lang-Image.png b/Tests/Images/Ja-Lang-Image.png new file mode 100644 index 00000000..3d1c6ae9 Binary files /dev/null and b/Tests/Images/Ja-Lang-Image.png differ diff --git a/Tests/OcrTests.cs b/Tests/OcrTests.cs index 996488be..c34afb0e 100644 --- a/Tests/OcrTests.cs +++ b/Tests/OcrTests.cs @@ -7,6 +7,7 @@ using Text_Grab; using Text_Grab.Interfaces; using Text_Grab.Models; +using Text_Grab.Properties; using Text_Grab.Utilities; using Windows.Globalization; @@ -90,6 +91,18 @@ BANK CHARGES 344 670 326 49% TOTAL EXPENDITURES 1,016,684 2,065,620 1,048,936 51% REVENUES OVERY(UNDER) EXPENDITURES $9,749 $0 $9,749 N/A"; + private const string JaTestExpectedResult = @"""くろ からだ しつ +黒ごまは体にいいです。タンバク質やカルシウムが +かみ くろ こうか +たくさんあります。髪を黒くする効果もあります。 +くろ あぶら はだ かみ りようり +黒ごま油は肌や髪に使います。料理にも使います。 +かゆ た からだ +お粥やデサートに入れます。でも、食べすきると体 +た +によくないです。少しすつ食べましよう。 +"""; + [WpfFact] public async Task OcrFontSampleImage() { @@ -268,6 +281,135 @@ public void GroupWrappedParagraphLines_CombinesWrappedLinesIntoParagraphBlocks() Assert.Equal("New paragraph.", groups[1].SingleLineText); } + private const string jaTestPath = @".\Images\Ja-Lang-Image.png"; + + // The reading-order-corrected OCR output for Ja-Lang-Image.png. Furigana ruby + // lines are still present inline (they are kept per the current line-ordering + // fix), but every line now appears in top-to-bottom / left-to-right reading + // order instead of the scrambled order the Windows OCR engine returns. + // + // JaTestExpectedResult (above) is the aspirational, fully-corrected target: + // furigana grouped per row with full-width spaces AND engine misreads fixed + // (からだ vs からた, こうか vs カ, ...). Reaching it needs more than ordering: + // furigana row grouping plus OCR error correction that recovers dakuten and + // small-kana the engine drops. This constant captures what is achievable today. + private const string JaReadingOrderResult = + "くろからたしつ黒ごまは体にいいです。タンバク質やカルシウムがかみ彡ろカたくさんあります。髪を黒くする効果もあります。くろあぶらはだかみりようり黒ごま油は肌や髪に使います。料理にも使います。かゆたからだお粥やデサ ー トに入れます。でも、食べすき、ると体たによくないです。少しすっ食べましよう。"; + + // With furigana removal enabled, the ruby-reading lines are dropped and only + // the main body text remains (still subject to the engine's own misreads and + // one stray mis-detected fragment "み彡" the geometry heuristic cannot catch). + private const string JaFuriganaRemovedResult = + "黒ごまは体にいいです。タンバク質やカルシウムがみ彡たくさんあります。髪を黒くする効果もあります。黒ごま油は肌や髪に使います。料理にも使います。お粥やデサ ー トに入れます。でも、食べすき、ると体によくないです。少しすっ食べましよう。"; + + [WpfFact] + public async Task OcrJapaneseImage_ReadingOrder_KeepsFuriganaWhenDisabled() + { + // Given + GlobalLang japanese = new("ja"); + + // Skip if the Japanese OCR language pack is not installed on this machine. + if (!Windows.Media.Ocr.OcrEngine.IsLanguageSupported(japanese.OriginalLanguage)) + return; + + Settings settings = AppUtilities.TextGrabSettings; + bool originalRemoveFurigana = settings.RemoveFurigana; + settings.RemoveFurigana = false; + + try + { + // When + string ocrTextResult = await OcrUtilities.OcrAbsoluteFilePathAsync( + FileUtilities.GetPathToLocalFile(jaTestPath), japanese); + + // Then furigana are kept, but every line is in natural reading order + // (top-to-bottom, left-to-right). + Assert.Equal(JaReadingOrderResult, ocrTextResult); + } + finally + { + settings.RemoveFurigana = originalRemoveFurigana; + } + } + + [WpfFact] + public async Task OcrJapaneseImage_RemovesFuriganaWhenEnabled() + { + // Given + GlobalLang japanese = new("ja"); + + if (!Windows.Media.Ocr.OcrEngine.IsLanguageSupported(japanese.OriginalLanguage)) + return; + + Settings settings = AppUtilities.TextGrabSettings; + bool originalRemoveFurigana = settings.RemoveFurigana; + settings.RemoveFurigana = true; + + try + { + // When + string ocrTextResult = await OcrUtilities.OcrAbsoluteFilePathAsync( + FileUtilities.GetPathToLocalFile(jaTestPath), japanese); + + // Then the furigana ruby lines are dropped, leaving the main text. + Assert.Equal(JaFuriganaRemovedResult, ocrTextResult); + } + finally + { + settings.RemoveFurigana = originalRemoveFurigana; + } + } + + [WpfFact] + public async Task InspectJapaneseOcrOutput() + { + // Exploration harness: dumps the raw OCR lines/words with their bounding + // boxes so we can see exactly what the Windows OCR engine returns for a + // furigana-heavy Japanese image, and how the current pipeline processes it. + GlobalLang japanese = new("ja"); + + if (!Windows.Media.Ocr.OcrEngine.IsLanguageSupported(japanese.OriginalLanguage)) + return; + + Bitmap testBitmap = new(FileUtilities.GetPathToLocalFile(jaTestPath)); + double scale = await OcrUtilities.GetIdealScaleFactorForOcrAsync(testBitmap, japanese); + Bitmap scaledBitmap = ImageMethods.ScaleBitmapUniform(testBitmap, scale); + IOcrLinesWords ocrResult = await OcrUtilities.GetOcrResultFromImageAsync(scaledBitmap, japanese); + + StringBuilder report = new(); + report.AppendLine($"scale factor: {scale:0.###}"); + report.AppendLine($"line count: {ocrResult.Lines.Length}"); + report.AppendLine(); + + for (int i = 0; i < ocrResult.Lines.Length; i++) + { + IOcrLine line = ocrResult.Lines[i]; + Windows.Foundation.Rect lb = line.BoundingBox; + report.AppendLine( + $"LINE {i,2} Y={lb.Y,7:0.0} H={lb.Height,6:0.0} X={lb.X,7:0.0} W={lb.Width,7:0.0} \"{line.Text}\""); + foreach (IOcrWord w in line.Words) + { + Windows.Foundation.Rect wb = w.BoundingBox; + report.AppendLine( + $" word Y={wb.Y,7:0.0} H={wb.Height,6:0.0} X={wb.X,7:0.0} W={wb.Width,7:0.0} \"{w.Text}\""); + } + } + + report.AppendLine(); + report.AppendLine("=== reading-flow ordered lines ==="); + foreach (IOcrLine line in OcrUtilities.OrderLinesForReadingFlow(ocrResult.Lines)) + report.AppendLine($" Y={line.BoundingBox.Y,7:0.0} X={line.BoundingBox.X,7:0.0} \"{line.Text}\""); + + report.AppendLine(); + report.AppendLine("=== BuildTextFromOcrLines (current pipeline output) ==="); + report.AppendLine(OcrUtilities.BuildTextFromOcrLines(japanese, ocrResult)); + + string outPath = Path.Combine(Path.GetTempPath(), "ja-ocr-report.txt"); + await File.WriteAllTextAsync(outPath, report.ToString(), new UTF8Encoding(true)); + System.Diagnostics.Debug.WriteLine(report.ToString()); + System.Diagnostics.Debug.WriteLine($"Report written to {outPath}"); + } + [WpfFact] public async Task ReadQrCode() { @@ -476,6 +618,363 @@ public async Task GetTesseractGitHubLanguage() File.Delete(tempFilePath); } + [Fact] + public void BuildTextFromOcrLines_FiltersFuriganaForJapanese() + { + // Given a Japanese line where the kanji 黒 is annotated with the small + // furigana くろ rendered directly above it. + FakeOcrLine line = new("くろ黒ごま", new Windows.Foundation.Rect(0, 0, 60, 30)) + { + Words = + [ + // Furigana: short and sitting above the kanji it annotates. + new FakeOcrWord("くろ", new Windows.Foundation.Rect(0, 0, 16, 8)), + // Main text: full-height single characters. + new FakeOcrWord("黒", new Windows.Foundation.Rect(0, 10, 20, 20)), + new FakeOcrWord("ご", new Windows.Foundation.Rect(20, 10, 20, 20)), + new FakeOcrWord("ま", new Windows.Foundation.Rect(40, 10, 20, 20)), + ] + }; + + FakeOcrLinesWords ocrResult = new() { Lines = [line] }; + + // When + string text = OcrUtilities.BuildTextFromOcrLines(new GlobalLang("ja"), ocrResult); + + // Then the furigana is dropped, leaving only the main text. + Assert.Equal("黒ごま", text); + } + + // ----- FilterFurigana unit tests (the geometry heuristic) ----- + + [Fact] + public void FilterFurigana_EmptyList_ReturnsEmpty() + { + List result = OcrUtilities.FilterFurigana([]); + + Assert.Empty(result); + } + + [Fact] + public void FilterFurigana_SingleWord_IsKept() + { + List words = [Word("黒", 0, 0, 20, 20)]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["黒"], result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_UniformHeights_KeepsAllInOrder() + { + // No word is small relative to the median, so nothing is furigana. + List words = + [ + Word("黒", 0, 0, 20, 20), + Word("ご", 20, 0, 20, 20), + Word("ま", 40, 0, 20, 20), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["黒", "ご", "ま"], result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_RemovesSmallWordAboveOverlappingKanji() + { + List words = + [ + Word("くろ", 0, 0, 16, 8), // furigana: short, sitting above + Word("黒", 0, 10, 20, 20), // kanji: taller, below, overlapping + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["黒"], result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_KeepsSmallWordWhenNotHorizontallyOverlapping() + { + // Small, but nowhere near a kanji horizontally, so it is real text. + List words = + [ + Word("くろ", 100, 0, 16, 8), + Word("黒", 0, 10, 20, 20), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["くろ", "黒"], result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_KeepsSmallWordBelowMainText() + { + // Furigana sits above its kanji; a small word BELOW a larger word is + // not furigana and must be kept. + List words = + [ + Word("黒", 0, 0, 20, 20), + Word("くろ", 0, 22, 16, 8), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["黒", "くろ"], result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_KeepsSmallWordWhenWordBelowIsNotLarger() + { + // A small word directly above another small word is not furigana: + // furigana requires a larger word (the kanji) beneath it. The two tall + // words only exist to raise the median height. + List words = + [ + Word("く", 0, 0, 8, 8), + Word("ろ", 0, 10, 8, 8), // below + overlapping, but also small + Word("本", 50, 0, 20, 20), + Word("語", 80, 0, 20, 20), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["く", "ろ", "本", "語"], result.Select(w => w.Text)); + } + + [Theory] + [InlineData("く", true)] // 1-char ruby is removed + [InlineData("くろ", true)] // 2-char ruby is removed + [InlineData("くろが", false)] // 3+ chars is treated as real text and kept + public void FilterFurigana_OnlyRemovesShortWords(string rubyText, bool removed) + { + List words = + [ + Word(rubyText, 0, 0, 16, 8), + Word("黒", 0, 10, 20, 20), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + string[] expected = removed ? ["黒"] : [rubyText, "黒"]; + Assert.Equal(expected, result.Select(w => w.Text)); + } + + [Fact] + public void FilterFurigana_RemovesMultipleFuriganaKeepingMainText() + { + List words = + [ + Word("くろ", 0, 0, 16, 8), + Word("黒", 0, 10, 20, 20), + Word("ごま", 20, 0, 16, 8), + Word("米", 20, 10, 20, 20), + ]; + + List result = OcrUtilities.FilterFurigana(words); + + Assert.Equal(["黒", "米"], result.Select(w => w.Text)); + } + + // ----- BuildTextFromOcrLines integration (language gating) ----- + + [Fact] + public void BuildTextFromOcrLines_JapaneseWithoutFurigana_IsUnchanged() + { + FakeOcrLine line = new("黒ごま", new Windows.Foundation.Rect(0, 0, 60, 20)) + { + Words = + [ + Word("黒", 0, 0, 20, 20), + Word("ご", 20, 0, 20, 20), + Word("ま", 40, 0, 20, 20), + ] + }; + FakeOcrLinesWords ocrResult = new() { Lines = [line] }; + + string text = OcrUtilities.BuildTextFromOcrLines(new GlobalLang("ja"), ocrResult); + + Assert.Equal("黒ごま", text); + } + + [Fact] + public void BuildTextFromOcrLines_ChineseText_JoinsWithoutSpaces() + { + FakeOcrLine line = new("中文", new Windows.Foundation.Rect(0, 0, 40, 20)) + { + Words = + [ + Word("中", 0, 0, 20, 20), + Word("文", 20, 0, 20, 20), + ] + }; + FakeOcrLinesWords ocrResult = new() { Lines = [line] }; + + string text = OcrUtilities.BuildTextFromOcrLines(new GlobalLang("zh-Hans"), ocrResult); + + Assert.Equal("中文", text); + } + + [Fact] + public void BuildTextFromOcrLines_FiltersRubyTextForChinese() + { + // The same small-ruby heuristic also runs for Chinese, another + // non-space-joining language (e.g. bopomofo above a character). + FakeOcrLine line = new("ㄓ中文", new Windows.Foundation.Rect(0, 0, 40, 30)) + { + Words = + [ + Word("ㄓ", 0, 0, 8, 8), + Word("中", 0, 10, 20, 20), + Word("文", 20, 10, 20, 20), + ] + }; + FakeOcrLinesWords ocrResult = new() { Lines = [line] }; + + string text = OcrUtilities.BuildTextFromOcrLines(new GlobalLang("zh-Hans"), ocrResult); + + Assert.Equal("中文", text); + } + + [Fact] + public void BuildTextFromOcrLines_SpaceJoiningLanguage_DoesNotFilterFurigana() + { + // For space-joining languages the whole line text is used verbatim, so + // the furigana heuristic never runs, even with a tiny word present. + Settings settings = AppUtilities.TextGrabSettings; + bool originalParagraphDetection = settings.ParagraphDetection; + bool originalCorrectErrors = settings.CorrectErrors; + settings.ParagraphDetection = false; + settings.CorrectErrors = false; + + try + { + FakeOcrLine line = new("Hello World", new Windows.Foundation.Rect(0, 0, 100, 30)) + { + Words = + [ + Word("x", 0, 0, 4, 4), // tiny word that would be furigana in CJK + Word("Hello", 0, 10, 50, 20), + Word("World", 55, 10, 50, 20), + ] + }; + FakeOcrLinesWords ocrResult = new() { Lines = [line] }; + + string text = OcrUtilities.BuildTextFromOcrLines(new GlobalLang("en-US"), ocrResult); + + Assert.Equal("Hello World" + System.Environment.NewLine, text); + } + finally + { + settings.ParagraphDetection = originalParagraphDetection; + settings.CorrectErrors = originalCorrectErrors; + } + } + + [Fact] + public void OrderLinesForReadingFlow_SortsRowsTopToBottomAndLeftToRight() + { + // Mimics the Windows OCR engine returning furigana ruby lines and a + // trailing fragment out of reading order (as seen with Ja-Lang-Image.png). + // Row 1 (y~0): furigana くろ + main-line reading, emitted out of x-order. + // Row 2 (y~30): the main text line. + FakeOcrLine furiganaRight = new("しつ", new Windows.Foundation.Rect(200, 0, 20, 8)); + FakeOcrLine furiganaLeft = new("くろ", new Windows.Foundation.Rect(0, 0, 20, 8)); + FakeOcrLine mainLine = new("黒ごま質", new Windows.Foundation.Rect(0, 30, 240, 20)); + + // Engine order is scrambled: right furigana, main line, then left furigana. + FakeOcrLinesWords ocrResult = new() + { + Lines = [furiganaRight, mainLine, furiganaLeft] + }; + + IReadOnlyList ordered = OcrUtilities.OrderLinesForReadingFlow(ocrResult.Lines); + + Assert.Equal(["くろ", "しつ", "黒ごま質"], ordered.Select(l => l.Text)); + } + + [Fact] + public void OrderLinesForReadingFlow_KeepsSeparateRowsInVerticalOrder() + { + // Two furigana rows and two main-text rows interleaved and shuffled must + // come back strictly top-to-bottom. + FakeOcrLine ruby2 = new("かみ", new Windows.Foundation.Rect(0, 100, 20, 8)); + FakeOcrLine main2 = new("髪", new Windows.Foundation.Rect(0, 130, 40, 20)); + FakeOcrLine ruby1 = new("くろ", new Windows.Foundation.Rect(0, 0, 20, 8)); + FakeOcrLine main1 = new("黒", new Windows.Foundation.Rect(0, 30, 40, 20)); + + FakeOcrLinesWords ocrResult = new() { Lines = [main2, ruby1, main1, ruby2] }; + + IReadOnlyList ordered = OcrUtilities.OrderLinesForReadingFlow(ocrResult.Lines); + + Assert.Equal(["くろ", "黒", "かみ", "髪"], ordered.Select(l => l.Text)); + } + + [Fact] + public void FilterFuriganaLines_RemovesShortLineAboveTallerOverlappingLine() + { + // A short furigana line sitting just above a taller kanji line that it + // overlaps horizontally is dropped. + FakeOcrLine furigana = new("くろ", new Windows.Foundation.Rect(0, 0, 40, 8)); + FakeOcrLine mainLine = new("黒ごま", new Windows.Foundation.Rect(0, 10, 120, 20)); + + FakeOcrLinesWords ocrResult = new() { Lines = [furigana, mainLine] }; + + IReadOnlyList result = OcrUtilities.FilterFuriganaLines(ocrResult.Lines); + + Assert.Equal(["黒ごま"], result.Select(l => l.Text)); + } + + [Fact] + public void FilterFuriganaLines_KeepsTwoBodyLinesOfSimilarHeight() + { + // Two normal body lines stacked vertically: neither is much shorter than + // the other, so nothing is treated as furigana. + FakeOcrLine top = new("黒ごまは体に", new Windows.Foundation.Rect(0, 0, 200, 20)); + FakeOcrLine bottom = new("たくさんあります", new Windows.Foundation.Rect(0, 26, 200, 20)); + + FakeOcrLinesWords ocrResult = new() { Lines = [top, bottom] }; + + IReadOnlyList result = OcrUtilities.FilterFuriganaLines(ocrResult.Lines); + + Assert.Equal(["黒ごまは体に", "たくさんあります"], result.Select(l => l.Text)); + } + + [Fact] + public void FilterFuriganaLines_KeepsShortLineNotHorizontallyOverlappingAnyKanji() + { + // A short line off to the side (no taller line beneath it) is real text. + FakeOcrLine shortSide = new("注", new Windows.Foundation.Rect(300, 0, 20, 8)); + FakeOcrLine mainLine = new("黒ごま", new Windows.Foundation.Rect(0, 10, 120, 20)); + + FakeOcrLinesWords ocrResult = new() { Lines = [shortSide, mainLine] }; + + IReadOnlyList result = OcrUtilities.FilterFuriganaLines(ocrResult.Lines); + + Assert.Equal(["注", "黒ごま"], result.Select(l => l.Text)); + } + + [Fact] + public void FilterFuriganaLines_KeepsShortLineWhenGapIsTooLarge() + { + // Short line far above a taller line is a separate heading/body line, not + // a hugging ruby annotation, so it is kept. + FakeOcrLine shortHeading = new("メモ", new Windows.Foundation.Rect(0, 0, 40, 8)); + FakeOcrLine mainLine = new("黒ごま", new Windows.Foundation.Rect(0, 60, 120, 20)); + + FakeOcrLinesWords ocrResult = new() { Lines = [shortHeading, mainLine] }; + + IReadOnlyList result = OcrUtilities.FilterFuriganaLines(ocrResult.Lines); + + Assert.Equal(["メモ", "黒ごま"], result.Select(l => l.Text)); + } + + private static FakeOcrWord Word(string text, double x, double y, double width, double height) + => new(text, new Windows.Foundation.Rect(x, y, width, height)); + private sealed class FakeOcrLinesWords : IOcrLinesWords { public string Text { get; set; } = string.Empty; @@ -499,4 +998,17 @@ public FakeOcrLine(string text, Windows.Foundation.Rect boundingBox) public Windows.Foundation.Rect BoundingBox { get; set; } } + + private sealed class FakeOcrWord : IOcrWord + { + public FakeOcrWord(string text, Windows.Foundation.Rect boundingBox) + { + Text = text; + BoundingBox = boundingBox; + } + + public string Text { get; set; } + + public Windows.Foundation.Rect BoundingBox { get; set; } + } } diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 19ae399f..800d7bab 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -58,6 +58,9 @@ PreserveNewest + + PreserveNewest + diff --git a/Text-Grab/App.config b/Text-Grab/App.config index 98a9942b..c5ff66fd 100644 --- a/Text-Grab/App.config +++ b/Text-Grab/App.config @@ -268,6 +268,9 @@ True + + True + diff --git a/Text-Grab/Pages/LanguageSettings.xaml b/Text-Grab/Pages/LanguageSettings.xaml index 96412571..4eb0c13a 100644 --- a/Text-Grab/Pages/LanguageSettings.xaml +++ b/Text-Grab/Pages/LanguageSettings.xaml @@ -72,6 +72,47 @@ Content="Learn more about Windows AI Foundry" NavigateUri="https://learn.microsoft.com/en-us/windows/ai/apis/" /> + + + Furigana are the small kana readings printed above kanji in Japanese (and similar ruby + annotations in Chinese). The OCR engine reads them as extra text, which clutters the + result. When enabled, Text Grab detects those small ruby lines by their size and position + and removes them, keeping only the main text. + + + + Remove furigana (ruby text) from Japanese and Chinese OCR + + + + + This is a work in progress. Furigana detection is based on a size-and-position + heuristic, so it can occasionally miss a reading or remove text it shouldn't. I'm + actively trying to improve how Text Grab handles Japanese and Chinese. If you have + experience with CJK text, OCR, or furigana and would be willing to help, please email + me at + joe@joefinapps.com. Thank you! + + + True + + True + diff --git a/Text-Grab/Utilities/OcrUtilities.cs b/Text-Grab/Utilities/OcrUtilities.cs index f3b65301..3976c137 100644 --- a/Text-Grab/Utilities/OcrUtilities.cs +++ b/Text-Grab/Utilities/OcrUtilities.cs @@ -69,10 +69,17 @@ public static void GetTextFromOcrLine(this IOcrLine ocrLine, bool isSpaceJoining } else { + // For CJK languages, filter out likely furigana (small ruby-text + // characters above the main text) before merging the words. This is + // opt-in via the RemoveFurigana setting. + IEnumerable words = DefaultSettings.RemoveFurigana + ? FilterFurigana([.. ocrLine.Words]) + : ocrLine.Words; + bool isFirstWord = true; bool isPrevWordSpaceJoining = false; - foreach (IOcrWord ocrWord in ocrLine.Words) + foreach (IOcrWord ocrWord in words) { string wordString = ocrWord.Text; @@ -95,6 +102,62 @@ public static void GetTextFromOcrLine(this IOcrLine ocrLine, bool isSpaceJoining text.ReplaceGreekOrCyrillicWithLatin(); } + /// + /// Removes words that are likely furigana: small ruby-text characters + /// rendered above the main text in Japanese. A word is treated as furigana + /// when it is noticeably shorter than the line's median word height and sits + /// directly above a larger word that overlaps it horizontally. + /// + internal static List FilterFurigana(List words) + { + if (words.Count == 0) + return words; + + // Furigana is typically around half the height of the main text. + List heights = [.. words.Select(w => w.BoundingBox.Height).OrderBy(h => h)]; + double medianHeight = heights[heights.Count / 2]; + double furiganaThreshold = medianHeight * 0.6; + + List filteredWords = []; + + for (int i = 0; i < words.Count; i++) + { + IOcrWord word = words[i]; + bool isProbablyFurigana = false; + + if (word.BoundingBox.Height < furiganaThreshold) + { + // Only treat it as furigana when a larger word sits below it and + // overlaps horizontally (i.e. the kanji it annotates). + for (int j = 0; j < words.Count; j++) + { + if (i == j) + continue; + + IOcrWord otherWord = words[j]; + + bool isBelow = otherWord.BoundingBox.Top > word.BoundingBox.Bottom; + bool overlapsHorizontally = !(otherWord.BoundingBox.Right < word.BoundingBox.Left + || otherWord.BoundingBox.Left > word.BoundingBox.Right); + bool isLarger = otherWord.BoundingBox.Height > furiganaThreshold; + + if (isBelow && overlapsHorizontally && isLarger) + { + isProbablyFurigana = word.Text.Length <= 2; + break; + } + } + } + + if (!isProbablyFurigana) + filteredWords.Add(word); + } + + // If everything was filtered, fall back to the original words to avoid + // dropping the whole line. + return filteredWords.Count > 0 ? filteredWords : words; + } + public static async Task GetTextFromAbsoluteRectAsync( Rect rect, ILanguage language, @@ -538,7 +601,23 @@ .. GroupWrappedParagraphLines( } else { - foreach (IOcrLine ocrLine in lines) + // Windows OCR returns CJK lines - especially furigana ruby lines and + // stray fragments - in an order that does not follow the page's reading + // flow, so re-sort by geometry (top-to-bottom, then left-to-right) + // before joining. Space-joining languages keep the engine order because + // paragraph detection above already handles their layout. + IReadOnlyList orderedLines = isSpaceJoiningOCRLang + ? lines + : OrderLinesForReadingFlow(lines); + + // Windows OCR emits furigana (Japanese ruby readings) as their own + // short lines sitting directly above the kanji they annotate, so the + // word-level filter above never catches them. Drop those whole lines + // when furigana removal is enabled. + if (!isSpaceJoiningOCRLang && DefaultSettings.RemoveFurigana) + orderedLines = FilterFuriganaLines(orderedLines); + + foreach (IOcrLine ocrLine in orderedLines) ocrLine.GetTextFromOcrLine(isSpaceJoiningOCRLang, text); } @@ -548,6 +627,108 @@ .. GroupWrappedParagraphLines( return text.ToString(); } + /// + /// Re-orders OCR lines into natural reading flow: groups lines that share a + /// horizontal row (their vertical extents overlap), orders rows top-to-bottom, + /// and orders the lines within each row left-to-right. Windows OCR frequently + /// returns CJK lines out of order (furigana above kanji, trailing fragments), + /// which scrambles the concatenated text without this pass. + /// + internal static IReadOnlyList OrderLinesForReadingFlow(IReadOnlyList lines) + { + if (lines.Count <= 1) + return lines; + + // Stable sort by the top edge so rows are discovered top-to-bottom. + List byTop = [.. lines.OrderBy(line => line.BoundingBox.Top)]; + + List> rows = []; + double currentRowTop = 0; + double currentRowBottom = 0; + + foreach (IOcrLine line in byTop) + { + Windows.Foundation.Rect box = line.BoundingBox; + + if (rows.Count > 0) + { + double overlap = Math.Min(currentRowBottom, box.Bottom) - Math.Max(currentRowTop, box.Top); + double minHeight = Math.Min(currentRowBottom - currentRowTop, box.Height); + + // A line joins the current row when it overlaps the row's vertical + // band by more than half of the shorter of the two heights. + if (minHeight > 0 && overlap > minHeight * 0.5) + { + rows[^1].Add(line); + currentRowTop = Math.Min(currentRowTop, box.Top); + currentRowBottom = Math.Max(currentRowBottom, box.Bottom); + continue; + } + } + + rows.Add([line]); + currentRowTop = box.Top; + currentRowBottom = box.Bottom; + } + + List ordered = []; + foreach (List row in rows) + ordered.AddRange(row.OrderBy(line => line.BoundingBox.Left)); + + return ordered; + } + + /// + /// Removes whole OCR lines that are likely furigana: short ruby-reading lines + /// that sit directly above a substantially taller line overlapping them + /// horizontally (the kanji they annotate). Windows OCR returns furigana as + /// their own lines, so this complements the word-level . + /// The heuristic is intentionally conservative and geometry-only; it can miss + /// mis-detected readings and is offered as an opt-in, experimental setting. + /// + internal static IReadOnlyList FilterFuriganaLines(IReadOnlyList lines) + { + if (lines.Count < 2) + return lines; + + List kept = []; + + for (int i = 0; i < lines.Count; i++) + { + Windows.Foundation.Rect box = lines[i].BoundingBox; + bool isFurigana = false; + + for (int j = 0; j < lines.Count; j++) + { + if (i == j) + continue; + + Windows.Foundation.Rect other = lines[j].BoundingBox; + + bool isBelow = other.Top >= box.Bottom; + bool overlapsHorizontally = !(other.Right < box.Left || other.Left > box.Right); + // The annotated kanji is markedly taller than its reading. + bool isSubstantiallyTaller = other.Height > box.Height * 1.4; + // Ruby text hugs the top of its character; a large vertical gap + // means these are separate lines of body text, not a reading. + bool isCloseAbove = other.Top - box.Bottom < box.Height; + + if (isBelow && overlapsHorizontally && isSubstantiallyTaller && isCloseAbove) + { + isFurigana = true; + break; + } + } + + if (!isFurigana) + kept.Add(lines[i]); + } + + // Never drop everything - fall back to the input if the heuristic would + // erase the whole result. + return kept.Count > 0 ? kept : lines; + } + internal static bool ShouldUseParagraphDetection(bool isSpaceJoiningLanguage, bool isTableMode = false) { return DefaultSettings.ParagraphDetection && isSpaceJoiningLanguage && !isTableMode;