Core: Improve CJK emphasis normalization safety and image link migration (#378)

src/co_op_translator/core/llm/markdown_translator.py

@@ -12,6 +12,7 @@ from co_op_translator.utils.llm.markdown_utils import (
     _read_language_prompt_template,
     replace_code_blocks,
     restore_code_blocks,
+    normalize_cjk_emphasis_markers,
     SPLIT_DELIMITER,
 )
 from co_op_translator.utils.llm.code_comment_translator import (
@@ -156,7 +157,12 @@ class MarkdownTranslator(ABC):
         results = await self._run_prompts_sequentially(prompts, md_file_path)
         translated_content = "\n".join(results)
 
-        # Step 4: Restore the code blocks and inline code from placeholders
+        # Step 4: Normalize emphasis markers for CJK scripts to improve renderer compatibility
+        translated_content = normalize_cjk_emphasis_markers(
+            translated_content, language_code=language_code
+        )
+
+        # Step 4.5: Restore the code blocks and inline code from placeholders
         translated_content = restore_code_blocks(translated_content, placeholder_map)
 
         # Step 5: Update links

src/co_op_translator/core/project/translation_manager.py

@@ -646,10 +646,11 @@ class TranslationManager:
         all_errors = []
 
         try:
+            rename_map: dict[str, str] = {}
+            migrated_image_count = 0
+
             if "images" in self.translation_types:
                 # Migrate legacy translated image filenames and update markdown/notebook links
-                rename_map: dict[str, str] = {}
-                migrated_image_count = 0
 
                 try:
                     rename_map = migrate_translated_image_filenames(
@@ -706,6 +707,38 @@ class TranslationManager:
                     "Skipping translated image migration because image translation is disabled"
                 )
 
+            try:
+                # Always run link migration to rewrite legacy flattened links in content,
+                # even when no files were moved (empty rename_map)
+                migrated_md = self.directory_manager.migrate_markdown_image_links(
+                    rename_map
+                )
+                migrated_nb = self.directory_manager.migrate_notebook_image_links(
+                    rename_map
+                )
+                logger.info(
+                    "Migrated %d image files and updated %d markdown and %d notebook files",
+                    migrated_image_count,
+                    migrated_md,
+                    migrated_nb,
+                )
+            except Exception as e:
+                logger.warning(f"Image link migration skipped: {e}")
+
+            # As a safety net, canonicalize any remaining alias-based language dir segments in links
+            try:
+                md_fix, nb_fix = canonicalize_image_links_in_translations(
+                    self.translations_dir, self.image_dir
+                )
+                if md_fix or nb_fix:
+                    logger.info(
+                        "Canonicalized image links in %d markdown and %d notebooks",
+                        md_fix,
+                        nb_fix,
+                    )
+            except Exception as e:
+                logger.warning(f"Image link canonicalization skipped: {e}")
+
             # Clean up files no longer needed in target directories
             logger.info("Removing orphaned files...")
             with tqdm(total=1, desc="🧹 Cleaning orphaned files") as cleanup_progress:

src/co_op_translator/utils/common/file_utils.py

@@ -124,8 +124,13 @@ def update_readme_languages_table(
             else f"{repo_url_value}.git"
         )
 
-        # Handle `<repo_url>.git` first to avoid double-appending `.git`
+        # Handle `<repo_url>` placeholders before appending repo name snippets
         template = template.replace("<repo_url>.git", repo_url_with_git)
+        template = template.replace("<repo_url>", repo_url_value)
+
+        # Replace generic GitHub snippet placeholders when present in template
+        template = template.replace("https://github.com/*****.git", repo_url_with_git)
+
         try:
             tail = repo_url_without_git.rstrip("/").split("/")[-1]
             repo_name_value = tail[:-4] if tail.endswith(".git") else tail

src/co_op_translator/utils/llm/markdown_utils.py

@@ -24,12 +24,176 @@ from co_op_translator.utils.common.file_utils import (
     get_filename_and_extension,
     map_original_to_translated,
 )
+from co_op_translator.utils.common.lang_utils import normalize_language_code
 
 logger = logging.getLogger(__name__)
 
 
 SPLIT_DELIMITER = "\n\n===SYSTEM_USER_SPLIT===\n\n"
 
+CJK_CHAR_CLASS = r"\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af"
+CJK_EMPHASIS_LANGUAGE_PREFIXES = ("ja", "ko", "zh")
+_CJK_CHAR_RE = re.compile(rf"[{CJK_CHAR_CLASS}]")
+_CJK_FULL_TEXT_RE = re.compile(rf"^[{CJK_CHAR_CLASS}]+$")
+
+# Inner emphasis text must not contain '*' so a match cannot bleed into
+# neighboring emphasis regions.
+_EMPHASIS_INNER_TEXT_PATTERN = r"[^\n*]+?"
+
+
+def _build_cjk_emphasis_pattern(delim: str) -> re.Pattern[str]:
+    escaped = re.escape(delim)
+    return re.compile(
+        rf"(?<!\*){escaped}(?P<text>{_EMPHASIS_INNER_TEXT_PATTERN}){escaped}(?!\*)"
+    )
+
+
+_CJK_EMPHASIS_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    (_build_cjk_emphasis_pattern("***"), "<strong><em>{text}</em></strong>"),
+    (_build_cjk_emphasis_pattern("**"), "<strong>{text}</strong>"),
+    (_build_cjk_emphasis_pattern("*"), "<em>{text}</em>"),
+]
+
+
+def _apply_cjk_emphasis_pattern(
+    segment: str, pattern: re.Pattern[str], html_template: str
+) -> str:
+    def _replace(match: re.Match[str]) -> str:
+        inner_text = match.group("text")
+        if not inner_text:
+            return match.group(0)
+
+        source = match.string
+        start, end = match.span()
+        left_char = source[start - 1] if start > 0 else ""
+        right_char = source[end] if end < len(source) else ""
+        left_is_cjk = bool(_CJK_CHAR_RE.fullmatch(left_char))
+        right_is_cjk = bool(_CJK_CHAR_RE.fullmatch(right_char))
+        pure_cjk_inner = bool(_CJK_FULL_TEXT_RE.fullmatch(inner_text))
+
+        if left_is_cjk or right_is_cjk or pure_cjk_inner:
+            return html_template.format(text=inner_text)
+        return match.group(0)
+
+    return pattern.sub(_replace, segment)
+
+
+def _collect_inline_code_spans_with_markdown_ast(content: str) -> list[tuple[int, int]]:
+    """Collect absolute character ranges for inline code spans using Markdown AST context."""
+    md = MarkdownIt("commonmark")
+    tokens = md.parse(content)
+
+    lines = content.splitlines(keepends=True)
+    offsets = [0]
+    for ln in lines:
+        offsets.append(offsets[-1] + len(ln))
+
+    inline_spans: list[tuple[int, int]] = []
+
+    for tok in tokens:
+        if tok.type != "inline" or not tok.map or not tok.children:
+            continue
+
+        if not any(child.type == "code_inline" for child in tok.children):
+            continue
+
+        start_line, end_line = tok.map
+        seg_start = offsets[start_line]
+        seg_end = offsets[end_line]
+        segment = content[seg_start:seg_end]
+
+        idx = 0
+        while idx < len(segment):
+            if segment[idx] != "`":
+                idx += 1
+                continue
+
+            open_len = 1
+            while idx + open_len < len(segment) and segment[idx + open_len] == "`":
+                open_len += 1
+
+            close_idx = idx + open_len
+            while close_idx < len(segment):
+                if segment[close_idx] != "`":
+                    close_idx += 1
+                    continue
+
+                run_len = 1
+                while (
+                    close_idx + run_len < len(segment)
+                    and segment[close_idx + run_len] == "`"
+                ):
+                    run_len += 1
+
+                if run_len == open_len:
+                    inline_spans.append(
+                        (seg_start + idx, seg_start + close_idx + run_len)
+                    )
+                    idx = close_idx + run_len
+                    break
+
+                close_idx += run_len
+            else:
+                idx += open_len
+
+    inline_spans.sort(key=lambda x: x[0])
+    return inline_spans
+
+
+def normalize_cjk_emphasis_markers(
+    content: str,
+    language_code: str | None = None,
+    enabled_language_prefixes: tuple[str, ...] = CJK_EMPHASIS_LANGUAGE_PREFIXES,
+) -> str:
+    """Normalize emphasis markup around CJK text for renderer compatibility.
+
+    Some Markdown renderers fail to apply `*`/`**` emphasis when delimiters are
+    directly adjacent to CJK characters. To preserve visual intent without adding
+    visible spaces, convert those cases into equivalent HTML tags.
+
+    Args:
+        content: Markdown text that may include emphasis markers near CJK chars.
+        language_code: Optional translation target language code.
+        enabled_language_prefixes: Language prefixes where normalization is enabled.
+
+    Returns:
+        Markdown text with CJK-adjacent emphasis markers converted to HTML tags.
+    """
+
+    if language_code:
+        normalized_language = normalize_language_code(language_code).lower()
+        if not any(
+            normalized_language == prefix
+            or normalized_language.startswith(f"{prefix}-")
+            for prefix in enabled_language_prefixes
+        ):
+            return content
+
+    if "*" not in content:
+        return content
+
+    def _normalize_text_segment(segment: str) -> str:
+        for pattern, html_template in _CJK_EMPHASIS_PATTERNS:
+            segment = _apply_cjk_emphasis_pattern(segment, pattern, html_template)
+        return segment
+
+    # Skip inline code spans so literal examples are never rewritten.
+    # Use markdown-it AST to scope scanning to inline regions.
+    output_parts: list[str] = []
+    cursor = 0
+    inline_code_spans = _collect_inline_code_spans_with_markdown_ast(content)
+
+    for start, end in inline_code_spans:
+        if start > cursor:
+            output_parts.append(_normalize_text_segment(content[cursor:start]))
+        output_parts.append(content[start:end])
+        cursor = end
+
+    if cursor < len(content):
+        output_parts.append(_normalize_text_segment(content[cursor:]))
+
+    return "".join(output_parts)
+
 
 def _read_language_prompt_template(language_code: str) -> str:
     """Read a language-specific prompt template from packaged templates.
@@ -203,24 +367,29 @@ def split_markdown_content(content: str, max_tokens: int, tokenizer) -> list:
                         current_length = 0
 
                         if line_tokens > max_tokens:
-                            words = line.split()
-                            word_chunk = []
-                            word_chunk_tokens = 0
+                            if "@@CODE_BLOCK" in line or "@@INLINE_CODE" in line:
+                                chunks.append(line)
+                            else:
+                                words = line.split()
+                                word_chunk = []
+                                word_chunk_tokens = 0
 
-                            for word in words:
-                                word_with_space = word + " "
-                                word_tokens = count_tokens(word_with_space, tokenizer)
+                                for word in words:
+                                    word_with_space = word + " "
+                                    word_tokens = count_tokens(
+                                        word_with_space, tokenizer
+                                    )
 
-                                if word_chunk_tokens + word_tokens <= max_tokens:
-                                    word_chunk.append(word_with_space)
-                                    word_chunk_tokens += word_tokens
-                                else:
+                                    if word_chunk_tokens + word_tokens <= max_tokens:
+                                        word_chunk.append(word_with_space)
+                                        word_chunk_tokens += word_tokens
+                                    else:
+                                        chunks.append("".join(word_chunk))
+                                        word_chunk = [word_with_space]
+                                        word_chunk_tokens = word_tokens
+
+                                if word_chunk:
                                     chunks.append("".join(word_chunk))
-                                    word_chunk = [word_with_space]
-                                    word_chunk_tokens = word_tokens
-
-                            if word_chunk:
-                                chunks.append("".join(word_chunk))
                         else:
                             current_chunk = [line]
                             current_length = line_tokens
@@ -262,7 +431,9 @@ def _group_lines_preserving_list_items(text: str) -> list[str]:
                     idx += 1
                     continue
 
-                if next_line.startswith((" ", "\t")) or list_item_pattern.match(next_line):
+                if next_line.startswith((" ", "\t")) or list_item_pattern.match(
+                    next_line
+                ):
                     block.append(next_line)
                     idx += 1
                     continue

tests/co_op_translator/utils/llm/test_markdown_utils.py

@@ -12,6 +12,7 @@ from co_op_translator.utils.llm.markdown_utils import (
     count_links_in_markdown,
     split_markdown_content,
     update_notebook_links,
+    normalize_cjk_emphasis_markers,
 )
 
 
@@ -206,6 +207,102 @@ def test_split_markdown_content_keeps_list_item_with_code_placeholder():
     assert any("- Step 1" in chunk and "@@CODE_BLOCK_0@@" in chunk for chunk in chunks)
 
 
+def test_normalize_cjk_emphasis_markers_for_italic_and_bold():
+    """CJK-adjacent emphasis markers should be normalized to HTML tags."""
+    content = "これは*重要*です。これは**太字**です。"
+
+    normalized = normalize_cjk_emphasis_markers(content)
+
+    assert "これは<em>重要</em>です" in normalized
+    assert "これは<strong>太字</strong>です" in normalized
+
+
+def test_normalize_cjk_emphasis_markers_keeps_non_cjk_markdown_emphasis():
+    """Non-CJK emphasis formatting should remain unchanged."""
+    content = "This is *important* and **bold** text."
+
+    normalized = normalize_cjk_emphasis_markers(content)
+
+    assert normalized == content
+
+
+def test_normalize_cjk_emphasis_markers_skips_non_cjk_target_language():
+    """Normalization should not run when target language is not CJK."""
+    content = "これは*重要*です。これは**太字**です。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="fr")
+
+    assert normalized == content
+
+
+def test_normalize_cjk_emphasis_markers_runs_for_zh_regional_codes():
+    """Normalization should run for regional Chinese codes (e.g., zh-TW)."""
+    content = "這是*重點*。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="zh-TW")
+
+    assert "這是<em>重點</em>。" == normalized
+
+
+def test_normalize_cjk_emphasis_markers_converts_bold_italic_triple_asterisk():
+    """Triple-asterisk emphasis should convert to combined strong+em tags."""
+    content = "これは***重要***です。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert "これは<strong><em>重要</em></strong>です。" == normalized
+
+
+def test_normalize_cjk_emphasis_markers_converts_one_sided_bold_italic_boundaries():
+    """Triple-asterisk emphasis should normalize with one-sided CJK boundaries."""
+    content = "Start ***重要*** and これは***Configure*** end。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert "<strong><em>重要</em></strong>" in normalized
+    assert "これは<strong><em>Configure</em></strong>" in normalized
+
+
+def test_normalize_cjk_emphasis_markers_converts_one_sided_cjk_boundaries():
+    """Emphasis should normalize when either left or right boundary is CJK."""
+    content = "Start **太字** and *強調*です。次にこれは**Bold** end。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert "<strong>太字</strong>" in normalized
+    assert "<em>強調</em>です" in normalized
+    assert "これは<strong>Bold</strong>" in normalized
+
+
+def test_normalize_cjk_emphasis_markers_does_not_convert_underscore_patterns():
+    """Underscore-delimited fragments should remain unchanged to avoid identifier mutations."""
+    content = "変数_name_を確認します。"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert normalized == content
+
+
+def test_normalize_cjk_emphasis_markers_skips_inline_code_spans():
+    """Inline code spans should not be rewritten by emphasis normalization."""
+    content = "説明 `漢*字*語` と本文の漢*字*語"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert "`漢*字*語`" in normalized
+    assert "本文の漢<em>字</em>語" in normalized
+
+
+def test_normalize_cjk_emphasis_markers_skips_multibacktick_inline_code_spans():
+    """Inline code with multi-backtick delimiters should also be preserved."""
+    content = "説明 ``漢`*字*`語`` と本文の漢*字*語"
+
+    normalized = normalize_cjk_emphasis_markers(content, language_code="ja")
+
+    assert "``漢`*字*`語``" in normalized
+    assert "本文の漢<em>字</em>語" in normalized
+
+
 @pytest.fixture
 def complex_dir_structure(tmp_path):
     """Create a more complex directory structure for testing nested paths."""