Better Japanese post-processing

Tests/Tests.csproj

@@ -58,6 +58,9 @@
         <Content Include="Images\toc.png">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         </Content>
+        <Content Include="Images\Ja-Lang-Image.png">
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </Content>
     </ItemGroup>
 
     <ItemGroup>

Text-Grab/App.config

@@ -268,6 +268,9 @@
             <setting name="ParagraphDetection" serializeAs="String">
                 <value>True</value>
             </setting>
+            <setting name="RemoveFurigana" serializeAs="String">
+                <value>True</value>
+            </setting>
             <setting name="GrabFrameWordGrouping" serializeAs="String">
                 <value />
             </setting>

Text-Grab/Pages/LanguageSettings.xaml

@@ -72,6 +72,47 @@
             Content="Learn more about Windows AI Foundry"
             NavigateUri="https://learn.microsoft.com/en-us/windows/ai/apis/" />
 
+        <TextBlock
+            Margin="0,24,0,6"
+            FontSize="18"
+            FontWeight="Bold"
+            Style="{StaticResource TextBodyNormal}"
+            Text="Japanese &amp; Chinese (CJK) Text" />
+        <TextBlock
+            Margin="0,0,0,8"
+            Style="{StaticResource TextBodyNormal}"
+            TextWrapping="Wrap">
+            Furigana are the small kana readings printed above kanji in Japanese (and similar ruby
+            annotations in Chinese). The OCR engine reads them as extra text, which clutters the
+            result. When enabled, Text Grab detects those small ruby lines by their size and position
+            and removes them, keeping only the main text.
+        </TextBlock>
+        <ui:ToggleSwitch
+            x:Name="RemoveFuriganaToggle"
+            Margin="0,0,0,6"
+            Checked="RemoveFuriganaToggle_Checked"
+            Unchecked="RemoveFuriganaToggle_Checked">
+            <TextBlock Style="{StaticResource TextBodyNormal}">
+                Remove furigana (ruby text) from Japanese and Chinese OCR
+            </TextBlock>
+        </ui:ToggleSwitch>
+        <Border
+            Margin="0,0,0,10"
+            Padding="12"
+            Background="#142A767E"
+            BorderBrush="#2A767E"
+            BorderThickness="1"
+            CornerRadius="6">
+            <TextBlock Style="{StaticResource TextBodyNormal}" TextWrapping="Wrap">
+                This is a work in progress. Furigana detection is based on a size-and-position
+                heuristic, so it can occasionally miss a reading or remove text it shouldn't. I'm
+                actively trying to improve how Text Grab handles Japanese and Chinese. If you have
+                experience with CJK text, OCR, or furigana and would be willing to help, please email
+                me at
+                <Hyperlink NavigateUri="mailto:joe@joefinapps.com" RequestNavigate="Hyperlink_RequestNavigate">joe@joefinapps.com</Hyperlink>. Thank you!
+            </TextBlock>
+        </Border>
+
         <TextBlock
             Margin="0,24,0,6"
             FontSize="18"

Text-Grab/Pages/LanguageSettings.xaml.cs

@@ -39,6 +39,8 @@ private async void Page_Loaded(object sender, RoutedEventArgs e)
         LoadWindowsLanguages();
         LoadUiAutomationSettings();
 
+        RemoveFuriganaToggle.IsChecked = DefaultSettings.RemoveFurigana;
+
         if (DefaultSettings.UseTesseract)
         {
             TesseractLanguagesStackPanel.Visibility = Visibility.Visible;
@@ -217,6 +219,16 @@ private void UiAutomationTraversalModeComboBox_SelectionChanged(object sender, S
         DefaultSettings.Save();
     }
 
+    private void RemoveFuriganaToggle_Checked(object sender, RoutedEventArgs e)
+    {
+        if (loadingLanguageSettings)
+            return;
+
+        DefaultSettings.RemoveFurigana = RemoveFuriganaToggle.IsChecked is true;
+        DefaultSettings.Save();
+        LanguageUtilities.InvalidateAllCaches();
+    }
+
     private void WindowsAiDescriptionEnabledToggle_Checked(object sender, RoutedEventArgs e)
     {
         if (loadingLanguageSettings)

Text-Grab/Properties/Settings.settings

@@ -263,6 +263,9 @@
     <Setting Name="ParagraphDetection" Type="System.Boolean" Scope="User">
       <Value Profile="(Default)">True</Value>
     </Setting>
+    <Setting Name="RemoveFurigana" Type="System.Boolean" Scope="User">
+      <Value Profile="(Default)">True</Value>
+    </Setting>
     <Setting Name="GrabFrameWordGrouping" Type="System.String" Scope="User">
       <Value Profile="(Default)" />
     </Setting>

Text-Grab/Utilities/OcrUtilities.cs

@@ -69,10 +69,17 @@ public static void GetTextFromOcrLine(this IOcrLine ocrLine, bool isSpaceJoining
         }
         else
         {
+            // For CJK languages, filter out likely furigana (small ruby-text
+            // characters above the main text) before merging the words. This is
+            // opt-in via the RemoveFurigana setting.
+            IEnumerable<IOcrWord> words = DefaultSettings.RemoveFurigana
+                ? FilterFurigana([.. ocrLine.Words])
+                : ocrLine.Words;
+
             bool isFirstWord = true;
             bool isPrevWordSpaceJoining = false;
 
-            foreach (IOcrWord ocrWord in ocrLine.Words)
+            foreach (IOcrWord ocrWord in words)
             {
                 string wordString = ocrWord.Text;
 
@@ -95,6 +102,62 @@ public static void GetTextFromOcrLine(this IOcrLine ocrLine, bool isSpaceJoining
             text.ReplaceGreekOrCyrillicWithLatin();
     }
 
+    /// <summary>
+    /// Removes words that are likely furigana: small ruby-text characters
+    /// rendered above the main text in Japanese. A word is treated as furigana
+    /// when it is noticeably shorter than the line's median word height and sits
+    /// directly above a larger word that overlaps it horizontally.
+    /// </summary>
+    internal static List<IOcrWord> FilterFurigana(List<IOcrWord> words)
+    {
+        if (words.Count == 0)
+            return words;
+
+        // Furigana is typically around half the height of the main text.
+        List<double> heights = [.. words.Select(w => w.BoundingBox.Height).OrderBy(h => h)];
+        double medianHeight = heights[heights.Count / 2];
+        double furiganaThreshold = medianHeight * 0.6;
+
+        List<IOcrWord> filteredWords = [];
+
+        for (int i = 0; i < words.Count; i++)
+        {
+            IOcrWord word = words[i];
+            bool isProbablyFurigana = false;
+
+            if (word.BoundingBox.Height < furiganaThreshold)
+            {
+                // Only treat it as furigana when a larger word sits below it and
+                // overlaps horizontally (i.e. the kanji it annotates).
+                for (int j = 0; j < words.Count; j++)
+                {
+                    if (i == j)
+                        continue;
+
+                    IOcrWord otherWord = words[j];
+
+                    bool isBelow = otherWord.BoundingBox.Top > word.BoundingBox.Bottom;
+                    bool overlapsHorizontally = !(otherWord.BoundingBox.Right < word.BoundingBox.Left
+                        || otherWord.BoundingBox.Left > word.BoundingBox.Right);
+                    bool isLarger = otherWord.BoundingBox.Height > furiganaThreshold;
+
+                    if (isBelow && overlapsHorizontally && isLarger)
+                    {
+                        isProbablyFurigana = word.Text.Length <= 2;
+                        break;
+                    }
+                }
+            }
+
+            if (!isProbablyFurigana)
+                filteredWords.Add(word);
+        }
+
+        // If everything was filtered, fall back to the original words to avoid
+        // dropping the whole line.
+        return filteredWords.Count > 0 ? filteredWords : words;
+    }
+
     public static async Task<string> GetTextFromAbsoluteRectAsync(
         Rect rect,
         ILanguage language,
@@ -538,7 +601,23 @@ .. GroupWrappedParagraphLines(
         }
         else
         {
-            foreach (IOcrLine ocrLine in lines)
+            // Windows OCR returns CJK lines - especially furigana ruby lines and
+            // stray fragments - in an order that does not follow the page's reading
+            // flow, so re-sort by geometry (top-to-bottom, then left-to-right)
+            // before joining. Space-joining languages keep the engine order because
+            // paragraph detection above already handles their layout.
+            IReadOnlyList<IOcrLine> orderedLines = isSpaceJoiningOCRLang
+                ? lines
+                : OrderLinesForReadingFlow(lines);
+
+            // Windows OCR emits furigana (Japanese ruby readings) as their own
+            // short lines sitting directly above the kanji they annotate, so the
+            // word-level filter above never catches them. Drop those whole lines
+            // when furigana removal is enabled.
+            if (!isSpaceJoiningOCRLang && DefaultSettings.RemoveFurigana)
+                orderedLines = FilterFuriganaLines(orderedLines);
+
+            foreach (IOcrLine ocrLine in orderedLines)
                 ocrLine.GetTextFromOcrLine(isSpaceJoiningOCRLang, text);
         }
 
@@ -548,6 +627,108 @@ .. GroupWrappedParagraphLines(
         return text.ToString();
     }
 
+    /// <summary>
+    /// Re-orders OCR lines into natural reading flow: groups lines that share a
+    /// horizontal row (their vertical extents overlap), orders rows top-to-bottom,
+    /// and orders the lines within each row left-to-right. Windows OCR frequently
+    /// returns CJK lines out of order (furigana above kanji, trailing fragments),
+    /// which scrambles the concatenated text without this pass.
+    /// </summary>
+    internal static IReadOnlyList<IOcrLine> OrderLinesForReadingFlow(IReadOnlyList<IOcrLine> lines)
+    {
+        if (lines.Count <= 1)
+            return lines;
+
+        // Stable sort by the top edge so rows are discovered top-to-bottom.
+        List<IOcrLine> byTop = [.. lines.OrderBy(line => line.BoundingBox.Top)];
+
+        List<List<IOcrLine>> rows = [];
+        double currentRowTop = 0;
+        double currentRowBottom = 0;
+
+        foreach (IOcrLine line in byTop)
+        {
+            Windows.Foundation.Rect box = line.BoundingBox;
+
+            if (rows.Count > 0)
+            {
+                double overlap = Math.Min(currentRowBottom, box.Bottom) - Math.Max(currentRowTop, box.Top);
+                double minHeight = Math.Min(currentRowBottom - currentRowTop, box.Height);
+
+                // A line joins the current row when it overlaps the row's vertical
+                // band by more than half of the shorter of the two heights.
+                if (minHeight > 0 && overlap > minHeight * 0.5)
+                {
+                    rows[^1].Add(line);
+                    currentRowTop = Math.Min(currentRowTop, box.Top);
+                    currentRowBottom = Math.Max(currentRowBottom, box.Bottom);
+                    continue;
+                }
+            }
+
+            rows.Add([line]);
+            currentRowTop = box.Top;
+            currentRowBottom = box.Bottom;
+        }
+
+        List<IOcrLine> ordered = [];
+        foreach (List<IOcrLine> row in rows)
+            ordered.AddRange(row.OrderBy(line => line.BoundingBox.Left));
+
+        return ordered;
+    }
+
+    /// <summary>
+    /// Removes whole OCR lines that are likely furigana: short ruby-reading lines
+    /// that sit directly above a substantially taller line overlapping them
+    /// horizontally (the kanji they annotate). Windows OCR returns furigana as
+    /// their own lines, so this complements the word-level <see cref="FilterFurigana"/>.
+    /// The heuristic is intentionally conservative and geometry-only; it can miss
+    /// mis-detected readings and is offered as an opt-in, experimental setting.
+    /// </summary>
+    internal static IReadOnlyList<IOcrLine> FilterFuriganaLines(IReadOnlyList<IOcrLine> lines)
+    {
+        if (lines.Count < 2)
+            return lines;
+
+        List<IOcrLine> kept = [];
+
+        for (int i = 0; i < lines.Count; i++)
+        {
+            Windows.Foundation.Rect box = lines[i].BoundingBox;
+            bool isFurigana = false;
+
+            for (int j = 0; j < lines.Count; j++)
+            {
+                if (i == j)
+                    continue;
+
+                Windows.Foundation.Rect other = lines[j].BoundingBox;
+
+                bool isBelow = other.Top >= box.Bottom;
+                bool overlapsHorizontally = !(other.Right < box.Left || other.Left > box.Right);
+                // The annotated kanji is markedly taller than its reading.
+                bool isSubstantiallyTaller = other.Height > box.Height * 1.4;
+                // Ruby text hugs the top of its character; a large vertical gap
+                // means these are separate lines of body text, not a reading.
+                bool isCloseAbove = other.Top - box.Bottom < box.Height;
+
+                if (isBelow && overlapsHorizontally && isSubstantiallyTaller && isCloseAbove)
+                {
+                    isFurigana = true;
+                    break;
+                }
+            }
+
+            if (!isFurigana)
+                kept.Add(lines[i]);
+        }
+
+        // Never drop everything - fall back to the input if the heuristic would
+        // erase the whole result.
+        return kept.Count > 0 ? kept : lines;
+    }
+
     internal static bool ShouldUseParagraphDetection(bool isSpaceJoiningLanguage, bool isTableMode = false)
     {
         return DefaultSettings.ParagraphDetection && isSpaceJoiningLanguage && !isTableMode;