diff --git a/docs/weekly-log/2026-05-16-batch-ingest.md b/docs/weekly-log/2026-05-16-batch-ingest.md new file mode 100644 index 0000000..f23e7cb --- /dev/null +++ b/docs/weekly-log/2026-05-16-batch-ingest.md @@ -0,0 +1,375 @@ +# 2026-05-16 β€” W3 batch ingest (#17) + +> Status: **πŸŽ‰πŸŽ‰πŸŽ‰πŸŽ‰ 8/8 100% 적재 + Aura 톡계 5μ’… + NED 3μ°¨ 진단 ν™•μ • β†’ λ°œν‘œ 핡심 발견 λ°•μ œ μ™„λ£Œ.** +> λ°œν‘œ λ©”μ‹œμ§€ ν™•μ •: **선택지 3 β€” 디버깅 자체λ₯Ό trade-off μΈμ‚¬μ΄νŠΈλ‘œ** +> ⭐⭐⭐ **μ§„μ§œ 보석 발견**: Entity μΆ”μΆœ 라벨 ν’ˆμ§ˆ 문제 (Company 라벨이 싀은 금육 metric) + +## πŸ”₯πŸ”₯πŸ”₯ 5/16 진단 3μ’… β€” λ°œν‘œ 핡심 발견 (17:39~17:43 μΈ‘μ •) + +### 진단 1: λ―Έλž˜μ—μ…‹ 4Q λ¬Έμ„œμ˜ Entity top 20 + +```cypher +MATCH (d:Document {filename: "λ―Έλž˜μ—μ…‹μ¦κΆŒ_4λΆ„κΈ°_μ‹€μ λ³΄κ³ μ„œ.pdf"}) + -[:HAS_SECTION]->(:Section)-[:CONTAINS_CHUNK]->(c:Chunk) + -[:MENTIONS]->(e:Entity) +RETURN labels(e)[1] AS type, e.name, count(c) AS chunk_count +ORDER BY chunk_count DESC LIMIT 20; +``` + +κ²°κ³Ό (top 17): + +| Type | e.name | chunk_count | +|---------|-----------------------------------------|-------------| +| Company | "곡λͺ¨λ°œν–‰μ•‘ 23μ‘° 7,050얡원 (2025λ…„ 10μ›”)" | **19** | +| Company | "μœ μƒμ¦μž 41,186" | 4 | +| Company | "μœ μƒμ¦μž 77,258" | 4 | +| Company | "κΈˆμœ΅μ§€μ£Όμ±„ λ°œν–‰κ·œλͺ¨ 및 비쀑..." | 3 | +| Company | "AAλ“±κΈ‰ 이상 νšŒμ‚¬μ±„ λ°œν–‰ 비쀑 73.0%" | 3 | +| Metric | "μœ μƒμ¦μž 47,034 (κΈ°μ€€: μ¦κΆŒμ‹ κ³ μ„œ..)" | 3 | +| Company | "μ‹œμ„€ λͺ©μ  νšŒμ‚¬μ±„ λ°œν–‰ 비쀑 10.7%" | 3 | +| Company | "κΈ°μ—…κ³΅κ°œ μ „λ…„λ™κΈ°λŒ€λΉ„ 증감앑..." | 2 | +| ... (λŒ€λΆ€λΆ„ Company 인데 금육 수치) | ... | + +**🚨 좩격적 발견**: +- λ―Έλž˜μ—μ…‹ 4Q λ³΄κ³ μ„œ 68μ²­ν¬μ—μ„œ **νšŒμ‚¬λͺ…이 단 ν•˜λ‚˜λ„ Company entity 둜 μΆ”μΆœ μ•ˆ 됨** +- "Company" 둜 λΆ„λ₯˜λœ entity 듀이 μ‹€μ œλ‘œλŠ” **금육 metric / λ°œν–‰ 규λͺ¨ / 비쀑** λ“± 수치 +- "곡λͺ¨λ°œν–‰μ•‘ 23μ‘° 7,050얡원" 이 Company 라벨 + 19 청크 μ–ΈκΈ‰ (κ°€μž₯ 많이!) +- LLM (Kimi) 이 **entity μΆ”μΆœ μ‹œ 라벨을 잘λͺ» λΆ€μ—¬**ν•˜κ³  있음 + +### 진단 2: 영문/λŒ€λͺ…사 νŒ¨ν„΄ 검색 + +```cypher +MATCH (e:Entity:Company) +WHERE e.name CONTAINS "Mirae" OR e.name CONTAINS "당사" + OR e.name CONTAINS "νšŒμ‚¬" OR e.name CONTAINS "κ·Έλ£Ή" +RETURN e.name, e.aliases, e.member_count ORDER BY e.member_count DESC LIMIT 20; +``` + +κ²°κ³Ό: **νšŒμ‚¬λͺ… 0건**. λͺ¨λ‘ "νšŒμ‚¬μ±„ 23μ‘°..." 같은 수치 데이터. + +β†’ "당사", "λ³Έ νšŒμ‚¬", "κ·Έλ£Ή" 같은 λŒ€λͺ…μ‚¬λ‘œ entity μΆ”μΆœ μ•ˆ 됨 (Kimi κ°€ λ¬΄μ‹œν•˜κ±°λ‚˜ λ‹€λ₯Έ μΉ΄ν…Œκ³ λ¦¬) +β†’ 영문 "Mirae" ν‘œκΈ°λ‘œλ„ μΆ”μΆœ μ•ˆ 됨 + +### 진단 3: 전체 Company entity top 20 (member_count 순) + +```cypher +MATCH (e:Entity:Company) +RETURN e.name, e.member_count, e.aliases +ORDER BY e.member_count DESC LIMIT 20; +``` + +κ²°κ³Ό (top 14): + +| Rank | e.name (Company 라벨) | member_count | +|------|---------------------------------------------------|--------------| +| 1 | "총계 κΈˆμ•‘ μ „λ…„λ™κΈ°λŒ€λΉ„ 증감앑..." (NED 4 ν‘œκΈ°) | **6** | +| 2 | "κΈˆμœ΅μ‚¬" | 3 | +| 3 | "μΌλ°˜κΈ°μ—…" | 3 | +| 4 | "νšŒμ‚¬μ±„ λ°œν–‰ 규λͺ¨ 23μ‘° 6,111얡원..." | 2 | +| 5 | "단기채무 12,060만 원 (2024λ…„ 1∼10μ›”)" | 2 | +| 6 | "νšŒμ‚¬μ±„ 23μ‘° 6,111얡원 (μ „μ›” λŒ€λΉ„ 16.6%↓...)" | 2 | +| 7 | "ν•œκ΅­μ˜ˆνƒκ²°μ œμ›" | 2 | +| 8 | "κΈˆμ•‘ μ „μ›”λŒ€λΉ„ 증감앑(증감λ₯ ) β–³135 (β–³41.8%)..." | 2 | +| ... | λͺ¨λ‘ 금육 metric λ˜λŠ” 일반 μΉ΄ν…Œκ³ λ¦¬ | ... | + +**🚨 더 좩격적 발견**: +- **186개 Company entity 쀑 μ§„μ§œ νšŒμ‚¬λͺ…은 거의 μ—†μŒ** +- 1~3μœ„κ°€ "총계 κΈˆμ•‘", "κΈˆμœ΅μ‚¬", "μΌλ°˜κΈ°μ—…" β€” λͺ¨λ‘ **λ²”μ£Όλͺ… λ˜λŠ” metric** +- "ν•œκ΅­μ˜ˆνƒκ²°μ œμ›" 이 7μœ„ β€” **μœ μΌν•˜κ²Œ μ§„μ§œ κΈ°κ΄€λͺ…** +- νšŒμ‚¬λͺ…이 entity 둜 거의 μΆ”μΆœ μ•ˆ 됨 β†’ λ―Έλž˜μ—μ…‹μ¦κΆŒ 빈 결과의 μ§„μ§œ 원인 + +### 🎯 진단 μ’…ν•© β€” μ§„μ§œ 발견 + +> **"GraphRAG νŒŒμ΄ν”„λΌμΈμ˜ μ •μ§ν•œ ν•œκ³„ 발견: +> +> λ―Έλž˜μ—μ…‹μ¦κΆŒ 자기 νšŒμ‚¬ λ³΄κ³ μ„œ 4개 (1Q~4Q, 150 청크) μ μž¬ν–ˆμ§€λ§Œ 'Company' entity 둜 단 ν•˜λ‚˜λ„ μž‘νžˆμ§€ μ•ŠμŒ. +> +> μ§„μ§œ 원인 β€” Kimi κ°€ entity μΆ”μΆœ μ‹œ 'Company' 라벨을 잘λͺ» λΆ€μ—¬: +> - 금육 metric ('곡λͺ¨λ°œν–‰μ•‘ 23μ‘° 7,050얡원') β†’ Company ❌ +> - λ²”μ£Όλͺ… ('κΈˆμœ΅μ‚¬', 'μΌλ°˜κΈ°μ—…') β†’ Company ❌ +> - νšŒμ‚¬λͺ… β†’ 거의 λͺ» 작음 ❌ +> +> 즉 GraphRAG 의 κ°€μΉ˜ (κ·Έλž˜ν”„ ꡬ쑰, MENTIONS 1,553) λŠ” κ²€μ¦λμœΌλ‚˜, +> **Entity μΆ”μΆœ ν’ˆμ§ˆ** 이 λ³„λ„μ˜ production challenge. +> LLM ν”„λ‘¬ν”„νŠΈ μ—”μ§€λ‹ˆμ–΄λ§ + ν›„μ²˜λ¦¬ 검증이 ν•„μˆ˜."** + +## 🎀 λ°œν‘œ μŠ¬λΌμ΄λ“œ λ©”μ‹œμ§€ β€” μ§„μ§œ 보석 ⭐⭐⭐ + +### Before (5/16 17:37 μΈ‘μ • 직후 μΆ”μ •) +> "NED 의 ν•œκ΅­μ–΄ νšŒμ‚¬λͺ… ν‘œκΈ° λ³€ν˜• challenge" + +### After (5/16 17:43 진단 3μ’… ν™•μ •) +> **"Entity μΆ”μΆœ 라벨 ν’ˆμ§ˆ challenge β€” Kimi LLM 이 'Company' 라벨에 νšŒμ‚¬λͺ…이 μ•„λ‹Œ 금육 metric 을 λΆ€μ—¬ν•˜λŠ” νŒ¨ν„΄ 발견. 186개 Company entity 쀑 μ§„μ§œ κΈ°κ΄€λͺ…은 1개. 자기 νšŒμ‚¬ λ³΄κ³ μ„œμ— 자기 νšŒμ‚¬λͺ…이 entity 둜 μ•ˆ μž‘νžˆλŠ” μ •μ§ν•œ ν•œκ³„."** + +### VectorRAG ↔ GraphRAG 보완 관계 증λͺ… + +| 질의 μœ ν˜• | VectorRAG | GraphRAG | +|--------------------------------------|-----------|----------| +| "당사 μ˜μ—…μ΄μ΅μ€?" (λŒ€λͺ…사 ν•΄μ†Œ) | β—Ž 청크 μœ μ‚¬λ„λ‘œ λ‹΅λ³€ | ❌ entity μ—†μŒ | +| "νšŒμ‚¬ X 와 Y 의 곡톡 λ¦¬μŠ€ν¬λŠ”?" | β–³ LLM ν›„μ²˜λ¦¬ ν•„μš” | β—Ž FACES_RISK κ·Έλž˜ν”„ | +| "금육 μ§€ν‘œ 평균 μΆ”μ΄λŠ”?" | β–³ 청크 λͺ¨μŒ | β—Ž HAS_METRIC κ·Έλž˜ν”„ | +| "이 λ‹΅λ³€μ˜ μΆœμ²˜λŠ”?" | β–³ 청크 ID | β—Ž Layer A 좔적 | + +β†’ **λ‘˜ λ‹€ ν•„μš”ν•œ 게 μ •λŸ‰ 증거둜 증λͺ…됨**. λ°œν‘œμ˜ μ§„μ§œ κ²°λ‘ . + +## πŸ†πŸ†πŸ†πŸ† 5/16 8/8 100% 적재 μ™„λ£Œ (16:05 ~ 17:33, μ•½ 76λΆ„) + +### 1μ°¨ 적재: 8λ¬Έμ„œ (16:05~17:19, 71λΆ„) β€” 7/8 성곡 + +| 파일 | 포맷 | νƒ€μž… | sec | chunks(T/B) | ent(rawβ†’grp) | rel | mentions | parse | extract | link | neo4j | total | ok | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| +| DS투자증ꢌ_μ‹œν™©λΆ„μ„_리포트.pdf | pdf | report | 6 | 52/0 | 103β†’60 | 32 | 102 | 8.4 | 143.5 | 28.5 | 39.0 | 219.4 | βœ… | +| κΈˆμœ΅κ°λ…μ›_..._μ§μ ‘κΈˆμœ΅_쑰달싀적.doc | doc | (unknown) | 0 | 0/0 | 0β†’0 | 0 | 0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | ❌ | +| λ†ν˜‘_2022λ…„_9월말_κΈ°μ€€_μ‚¬μ—…λ³΄κ³ μ„œ.hwp | **hwp** | filing | 1 | 11/0 | 56β†’56 | 0 | 56 | 2.5 | 26.4 | 12.4 | 15.3 | 56.6 | βœ… | +| λ―Έλž˜μ—μ…‹μ¦κΆŒ_1λΆ„κΈ°_μ‹€μ λ³΄κ³ μ„œ.pdf | pdf | ir | 1 | 28/0 | 131β†’120 | 17 | 131 | 0.2 | 47.0 | 36.1 | 36.9 | 120.3 | βœ… | +| λ―Έλž˜μ—μ…‹μ¦κΆŒ_2λΆ„κΈ°_μ‹€μ λ³΄κ³ μ„œ.pdf | pdf | ir | 1 | 27/0 | 121β†’116 | 52 | 120 | 0.2 | 53.8 | 29.6 | 38.4 | 121.9 | βœ… | +| λ―Έλž˜μ—μ…‹μ¦κΆŒ_3λΆ„κΈ°_μ‹€μ λ³΄κ³ μ„œ.pdf | pdf | ir | 1 | 27/0 | 149β†’145 | 42 | 149 | **1514.8** | 47.2 | 25.9 | 43.8 | 1631.7 | βœ… | +| λ―Έλž˜μ—μ…‹μ¦κΆŒ_4λΆ„κΈ°_μ‹€μ λ³΄κ³ μ„œ.pdf | pdf | ir | 30 | **68/6** | **335β†’261** | 55 | 329 | **1694.7** | 109.7 | 109.4 | 102.3 | 2016.0 | βœ… | +| ν•œν™”νˆ¬μžμ¦κΆŒ_두산λ°₯μΊ£_기업뢄석_리포트.pdf | pdf | report | 12 | 15/5 | 42β†’33 | 31 | 42 | 12.9 | 40.7 | 9.2 | 20.3 | 83.1 | βœ… | + +### 2μ°¨ 적재: DOC β†’ DOCX λ³€ν™˜ ν›„ (17:28~17:33, 5λΆ„) β€” 1/1 성곡 + +| 파일 | 포맷 | νƒ€μž… | sec | chunks(T/B) | ent(rawβ†’grp) | rel | mentions | parse | extract | link | neo4j | total | ok | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| +| κΈˆμœ΅κ°λ…μ›_..._μ§μ ‘κΈˆμœ΅_쑰달싀적.docx | **docx** | **disclosure** | 1 | **49/0** | **275β†’251** | 26 | **260** | 1.7 | 106.0 | 116.6 | 73.3 | 297.7 | βœ… | + +### πŸŽ‰ μ§„μ§œ μ΅œμ’… 8/8 100% 총합 + +| μ§€ν‘œ | 1μ°¨ (7λ¬Έμ„œ) | 2μ°¨ (1λ¬Έμ„œ) | **총합** | +|---|---|---|---| +| 성곡λ₯  | 7/8 (87.5%) | 1/1 (100%) | **8/8 (100%)** πŸŽ‰ | +| μ†Œμš” μ‹œκ°„ | 71λΆ„ | 5λΆ„ | **76λΆ„** | +| 청크 (text) | 228 | 49 | **277** | +| Raw entity | 937 | 275 | **1,212** | +| Grouped entity | 791 | 251 | **1,042** | +| NED compression | 0.84 | 0.91 | **0.86** | +| Relation | 229 | 26 | **255** | +| MENTIONS | 929 | 260 | **1,189** | + +## 🎯 5/16 8λ¬Έμ„œ Aura Cypher 톡계 (17:37 μΈ‘μ •) + +### Q1: λ…Έλ“œ 라벨별 (총 610 λ…Έλ“œ) +``` +Chunk 277 / Entity 261 / Section 53 / Table 11 / Document 8 +``` + +### Q2: 관계 νƒ€μž…λ³„ (총 2,451 관계) +``` +MENTIONS 1553 / CONTAINS_CHUNK 277 / HAS_METRIC 270 / NEXT 235 +HAS_SECTION 53 / FACES_RISK 34 / HAS_OUTLOOK 12 / CONTAINS_TABLE 11 +RECOMMENDED_FOR 6 (μƒˆ 관계 νƒ€μž…) +``` + +### Q3: Entity νƒ€μž…λ³„ 뢄포 +``` +Company 186 (1,220 mentions, but λŒ€λΆ€λΆ„ metric! β†’ 진단 3 μ°Έκ³ ) +Metric 69 (297 mentions) +Risk 24 (29 mentions) +Outlook 7 (7 mentions) +``` + +⚠️ **Company 186개 쀑 μ§„μ§œ νšŒμ‚¬λͺ…은 1개 ("ν•œκ΅­μ˜ˆνƒκ²°μ œμ›")** β€” 진단 3μ—μ„œ ν™•μ •. + +### Q4: λ―Έλž˜μ—μ…‹μ¦κΆŒ NED β€” 빈 κ²°κ³Ό +``` +MATCH (e:Entity:Company {name: "λ―Έλž˜μ—μ…‹μ¦κΆŒ"}) ... β†’ No records +``` +β†’ 진단 3μ’…μœΌλ‘œ 원인 ν™•μ •: **Kimi 라벨 λΆ€μ—¬ 문제**. + +### Q5: doc_type 뢄포 +``` +pdf+ir 4 (λ―Έλž˜μ—μ…‹ 1Q~4Q) / pdf+report 2 (DS투자증ꢌ + ν•œν™”) / hwp+filing 1 (λ†ν˜‘) / docx+disclosure 1 (κΈˆκ°μ›) +``` +**3 포맷 Γ— 4 doc_type** λ‹€μ–‘μ„±. + +## 🌟 λ°œν‘œ 자료 보석 μ’…ν•© (5/23 μŠ¬λΌμ΄λ“œ μ‹œλ“œ) + +### A. DOC β†’ DOCX λ³€ν™˜ + 재적재 성곡 ⭐⭐⭐ +LibreOffice headless λ³€ν™˜ β†’ 49 청크 + 251 entity + 260 MENTIONS. μ—”μ§€λ‹ˆμ–΄λ§ 사이클 μ™„μ„±. + +### B. HWP νŒŒμ„œ 정상 λ™μž‘ ⭐ +λ†ν˜‘ μ‚¬μ—…λ³΄κ³ μ„œ 11 청크 적재. + +### C. λ―Έλž˜μ—μ…‹ 4λΆ„κΈ° 풍뢀 ⭐ +68 청크 + 6 ν‘œ (Table 첫 적재) + 335β†’261 entity + 329 MENTIONS. + +### D. ⭐⭐⭐ Entity μΆ”μΆœ 라벨 ν’ˆμ§ˆ challenge (진단 3μ’… ν™•μ •!) +- 186개 Company 쀑 μ§„μ§œ νšŒμ‚¬λͺ… 1개 +- "곡λͺ¨λ°œν–‰μ•‘ 23μ‘° 7,050얡원" 같은 metric 이 Company 라벨 +- LLM (Kimi) 의 라벨 λΆ€μ—¬ ν•œκ³„ β†’ ν”„λ‘¬ν”„νŠΈ μ—”μ§€λ‹ˆμ–΄λ§ + ν›„μ²˜λ¦¬ ν•„μš” + +### E. VectorRAG ↔ GraphRAG 보완 관계 μ •λŸ‰ 증λͺ… +- 자기 νšŒμ‚¬ λ³΄κ³ μ„œμ—μ„œ νšŒμ‚¬λͺ… entity μ—†μŒ β†’ VectorRAG 의 λŒ€λͺ…사 ν•΄μ†Œ 강점 +- κ·Έλž˜ν”„ traversal 은 GraphRAG 의 강점 (HAS_METRIC 270, FACES_RISK 34) +- **λ‘˜ λ‹€ ν•„μš”ν•œ 게 μ •λŸ‰ 증거둜 증λͺ…됨** + +### F. LLM λΉ„κ²°μ •μ„± +- DS투자증ꢌ 2회 μΆ”μΆœ: 113β†’55 vs 103β†’60 + +### G. doc_type μžλ™ λΆ„λ₯˜ 4κ°€μ§€ +report / ir / filing / disclosure + +### H. SemanticChunker 본질적 λΉ„νš¨μœ¨ +820자 / 31.3초 β†’ USE_SIMPLE_CHUNKER fallback + +## βœ… 5/16 ν† μš”μΌ 검증 μ™„λ£Œ 사항 + +- βœ… **λ‹¨μœ„ ν…ŒμŠ€νŠΈ 11/11 PASS** +- βœ… **PR #45 생성** β€” 6 commits +- βœ… **8/8 적재 μ™„λ£Œ** β€” 76λΆ„, 100% +- βœ… **Aura Cypher 5μ’… μΈ‘μ •** β€” 17:37 +- βœ… **진단 3μ’… μ™„λ£Œ** β€” 17:39~17:43, Entity 라벨 ν’ˆμ§ˆ ν™•μ • +- βœ… **μƒˆ Aura `9b57188f`** + .env κ°±μ‹  +- βœ… **LibreOffice λ―Έμ„€μΉ˜ β†’ μˆ˜λ™ λ³€ν™˜ 검증** + +## 🎯 5/17(일) μ‹œμž‘ κ°€μ΄λ“œ β€” μ§„μ§œ 마무리 + +### Step 1 β€” PR #45 λ¨Έμ§€ + 이슈 #17 close (1λΆ„) + +https://github.com/TaskerJang/doc-graph-agent/pull/45 β†’ **Merge pull request** λ²„νŠΌ. + +λ¨Έμ§€ ν›„: +```cmd +cd C:\Users\taske\doc-graph-agent +git checkout dev +git pull origin dev +git branch -d feat/17-batch-ingest +``` + +### Step 2 β€” #18 W4 Text2Cypher μ‹œμž‘ + +```cmd +git checkout -b feat/18-text2cypher +``` + +이슈 #18 μš”κ΅¬μ‚¬ν•­: +- `retrieval/text2cypher.py` β€” LLM 기반 μžμ—°μ–΄ β†’ Cypher λ³€ν™˜ +- μŠ€ν‚€λ§ˆ ν”„λ‘¬ν”„νŠΈ β€” λ…Έλ“œ/관계 μ •μ˜ LLM μ£Όμž… +- μ•ˆμ „μž₯치 β€” read-only κ°•μ œ + LIMIT 100 +- κ²°κ³Ό νŒŒμ‹± β†’ μžμ—°μ–΄ λ‹΅λ³€ (LLM 2단계) +- 평가 μ…‹ 5개 질문 μ •μ„± 검증 + +**5/17 평가 μ…‹ 질문 후보** (8λ¬Έμ„œ κ·Έλž˜ν”„ μœ„μ—μ„œ): +1. "DS투자증ꢌ μ‹œν™©λΆ„μ„ λ¦¬ν¬νŠΈμ—μ„œ μ–ΈκΈ‰λœ entity 듀은? (νšŒμ‚¬+metric 혼재 검증)" +2. "λ―Έλž˜μ—μ…‹μ¦κΆŒ 4λΆ„κΈ° λ³΄κ³ μ„œμ˜ Table 은 λͺ‡ κ°œμΈκ°€?" +3. "λ³΄λ„μžλ£Œ(disclosure) μœ ν˜• λ¬Έμ„œμ˜ entity 듀은?" +4. "두산λ°₯μΊ£κ³Ό ν•¨κ»˜ μ–ΈκΈ‰λœ λ¦¬μŠ€ν¬κ°€ μžˆλŠ”κ°€?" +5. "전체 κ·Έλž˜ν”„μ—μ„œ κ°€μž₯ 많이 μ–ΈκΈ‰λœ Company 라벨 entity 5κ°œλŠ”? (μ‹€μ œ νšŒμ‚¬ vs metric λΉ„μœ¨ 확인)" + +5번이 μ§„μ§œ ν₯λ―Έλ‘œμ›€ β€” Text2Cypher κ°€ "Company" 라벨 κ²°κ³Όλ₯Ό μ‚¬μš©μžμ—κ²Œ 보여쀄 λ•Œ, 싀은 metric 이 μ„žμΈ κ²°κ³ΌλΌλŠ” challenge. + +--- + +## μž‘μ—… λ²”μœ„ + +PR #44 (#24 Opik 1단계) λ¨Έμ§€ 직후 μ§„ν–‰. 두산λ°₯μΊ£ 1청크 νŒŒμ΄ν”„λΌμΈ β†’ 8λ¬Έμ„œ 전체 ν™•μž₯. + +### λ³€κ²½ 사항 + +**Layer A 적재 μΆ”κ°€** (`kg/builder.py`): +- `build_layer_a()` β€” Document/Section/Chunk/Table MERGE + 4관계 +- `link_chunks_to_entities()` β€” Chunk β†’ Entity `[:MENTIONS]` + +**μ˜΅μ…˜ 3 chunk_id global prefix** (`kg/extractor.py`): +- `{chunk_id}__ent_001` ν˜•μ‹ + `make_global_id` / `parse_global_id` + +**일괄 처리 슀크립트** (`scripts/run_w3_batch.py`): +- 8λ¬Έμ„œ 순회 + stat + stdout ν‘œ + +**chunker.py USE_SIMPLE_CHUNKER fallback** (commit `8f0bcb1`): +- `RecursiveCharacterTextSplitter` λΆ„κΈ°, κΈ°λ³Έ OFF + +## 🎀 λ°œν‘œ λ©”μ‹œμ§€ ν™•μ • (5/23) β€” 선택지 3 + +> **"동일 chunker 둜 λΉ„κ΅ν•˜λ € ν–ˆμœΌλ‚˜, CPU ν™˜κ²½μ—μ„œ SemanticChunker κ°€ λΉ„ν˜„μ‹€μ μž„μ„ 발견. 운영 λΉ„μš©μ˜ trade-off λ₯Ό μ •λŸ‰ν™”ν•¨. μΆ”κ°€λ‘œ LLM Entity μΆ”μΆœμ˜ 라벨 ν’ˆμ§ˆ challenge 도 발견."** + +### λ°œν‘œ ꡬ쑰 (12μŠ¬λΌμ΄λ“œ) + +1. 문제 μ •μ˜ +2. SemanticChunker 발견 (820자/31초) +3. 뢄석 (LangChain sequential) +4. trade-off ν‘œ +5. fallback κ΅¬ν˜„ +6. **운영 데이터 (8/8 100%, 76λΆ„)** +7. **포맷 λ‹€μ–‘μ„± (PDF + HWP + DOCX, 4 doc_type)** +8. **production μ˜μ‘΄μ„± (LibreOffice)** ⭐ +9. **κ·Έλž˜ν”„ 톡계 (610 / 2,451 / 1,553)** ⭐ +10. **⭐⭐⭐ Entity 라벨 ν’ˆμ§ˆ challenge β€” 186 Company 쀑 μ§„μ§œ νšŒμ‚¬λͺ… 1개!** +11. **VectorRAG ↔ GraphRAG 보완 관계 μ •λŸ‰ 증λͺ…** ⭐ NEW! +12. μΈμ‚¬μ΄νŠΈ λ©”μ‹œμ§€ + +### "κ·Έλž˜μ„œ κ²°κ΅­ μ„±λŠ₯ λΉ„κ΅λŠ”?" 질문 λŒ€μ‘ + +λ‹΅λ³€ μ€€λΉ„: +- SemanticChunker 원본은 production ν™˜κ²½ ν›„ (Issue #46) +- chunk μ •μ±… 동일 μœ μ§€, Layer ꡬ쑰 효과 μ •λŸ‰ν™”: 1,189 MENTIONS, 0.86 compression +- 4κ°€μ§€ doc_type μžλ™ λΆ„λ₯˜ +- **Entity 라벨 ν’ˆμ§ˆ** β€” Kimi ν•œκ³„ 발견, ν”„λ‘¬ν”„νŠΈ μ—”μ§€λ‹ˆμ–΄λ§ + ν›„μ²˜λ¦¬ ν•„μš” +- **VectorRAG ↔ GraphRAG κ°€ 보완 관계** λΌλŠ” 게 μ •λŸ‰ 증λͺ…됨 + +## μ‹œν–‰μ°©μ˜€ λ°•μ œ + +5/16 ν† μš”μΌ 디버깅 μžμ‚° (= λ°œν‘œ μŠ¬λΌμ΄λ“œ μ‹œλ“œ): +- βœ… **bge-m3 정상** (108ms/λ¬Έμž₯) +- βœ… **PDF / OCR / Neo4j / SemanticChunker μ΄ˆκΈ°ν™” λͺ¨λ‘ 정상** +- ❌ **SemanticChunker.split_text 본질적 λΉ„νš¨μœ¨** β€” 820자/31초 +- βœ… **USE_SIMPLE_CHUNKER fallback** β€” commit `8f0bcb1` +- βœ… **8/8 적재 100%** + **Aura 톡계 5μ’…** + **진단 3μ’…** +- βœ… **HWP / DOCX νŒŒμ„œ 정상** +- βœ… **Table λ…Έλ“œ 첫 적재** (11개) +- βœ… **doc_type 4κ°€μ§€ μžλ™ λΆ„λ₯˜** +- ⚠️⚠️⚠️ **Entity 라벨 ν’ˆμ§ˆ challenge** β€” Kimi κ°€ Company 라벨에 metric λΆ€μ—¬ +- ⚠️ **NED ν•œκ΅­μ–΄ νšŒμ‚¬λͺ… λ³€ν˜•** β€” 자기 νšŒμ‚¬ λ³΄κ³ μ„œμ— 자기 νšŒμ‚¬λͺ… entity μ—†μŒ +- **Aura Free trial** β€” μƒˆ μΈμŠ€ν„΄μŠ€ `9b57188f` +- **LibreOffice ν™˜κ²½ μ˜μ‘΄μ„±** β€” production Docker κ°€μ΄λ“œ +- **MS Store python stub** β€” `uv run` 영ν–₯ μ—†μŒ + +### 미래 발견 (#46+) + +- **LibreOffice μžλ™ν™”** β€” production Docker 이미지 μΆ”κ°€ +- **OCR μ‹œκ°„ 폭증** β€” 3Q/4Q parse 1500초+ β†’ μΊμ‹œ λ˜λŠ” 라이브러리 ꡐ체 +- **OpenAI λ§ˆμ΄κ·Έλ ˆμ΄μ…˜** (#46) β€” SemanticChunker 원본 볡원 +- **Entity μΆ”μΆœ ν”„λ‘¬ν”„νŠΈ μ—”μ§€λ‹ˆμ–΄λ§** ⭐ NEW! β€” Kimi 의 Company 라벨 ν’ˆμ§ˆ κ°œμ„  +- **NED ν•œκ΅­μ–΄ ν›„μ²˜λ¦¬** β€” νšŒμ‚¬λͺ… μ •κ·œν™” μΆ”κ°€ 단계 + +### local_id 좩돌 β€” μ˜΅μ…˜ 3 으둜 ν•΄κ²° + +``` +adapter.parse_document() β†’ Document.sections[].chunks[].id ("doc1:c0001") + ↓ +chunks = [{doc_id, chunk_id, section, text}] + ↓ extract() β€” chunk_id prefix +ExtractedEntity.local_id = "doc1:c0001__ent_001" + ↓ link_entities() β€” 청크 κ°„ NED +EntityGroup.members[*].local_id = "doc1:c0001__ent_001" + ↓ link_chunks_to_entities() +parse_global_id("doc1:c0001__ent_001") β†’ ("doc1:c0001", "ent_001") + ↓ +(:Chunk {id:"doc1:c0001"}) -[:MENTIONS]-> (:Entity {group_id:"grp_001"}) +``` + +## λ°œν‘œ 자료 μ‹œλ“œ (5/23 용) + +| 자료 | μƒνƒœ | 좜처 | +|---|---|---| +| 1λ¬Έμ„œ β†’ 8/8 적재 운영 데이터 ν‘œ | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 Aura 톡계 5μ’… (8λ¬Έμ„œ, 17:37)** | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 진단 3μ’… β€” Entity 라벨 ν’ˆμ§ˆ** ⭐⭐⭐ | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ 17:43 | +| **🎯 Company 186 쀑 μ§„μ§œ νšŒμ‚¬λͺ… 1개 (ν•œκ΅­μ˜ˆνƒκ²°μ œμ›)** | βœ… μ™„λ£Œ | 진단 3 | +| **🎯 자기 νšŒμ‚¬ λ³΄κ³ μ„œμ— 자기 νšŒμ‚¬λͺ… μ—†μŒ** | βœ… μ™„λ£Œ | 진단 1+2 | +| **🎯 VectorRAG ↔ GraphRAG 보완 관계 μ •λŸ‰ 증λͺ…** ⭐ NEW! | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 doc_type Γ— format 4Γ—3 λ‹€μ–‘μ„±** | βœ… μ™„λ£Œ | Q5 | +| Layer A + Layer B + MENTIONS κ·Έλž˜ν”„ μ‹œκ°ν™” | ⏸️ Aura console 캑처 | μΌμš”μΌ | +| **🎯 LibreOffice μ˜μ‘΄μ„± 발견 + ν•΄μ†Œ 사이클** | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| `@track` 으둜 μžλ™ μˆ˜μ§‘λœ trace | βœ… Opik UI 확인 κ°€λŠ₯ | Opik UI | +| **🎯 λ―Έλž˜μ—μ…‹ 4λΆ„κΈ° β€” 68 청크 + 6 ν‘œ + 261 entity** | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 LLM λΉ„κ²°μ •μ„± β€” DS투자증ꢌ 2회 μΆ”μΆœ 차이** | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 SemanticChunker 본질적 λΉ„νš¨μœ¨ (820자/31초)** | βœ… μ™„λ£Œ | λ³Έ λ¬Έμ„œ | +| **🎯 USE_SIMPLE_CHUNKER fallback μ½”λ“œ** | βœ… μ™„λ£Œ | commit `8f0bcb1` | + +## κ΄€λ ¨ + +- PR: #45 (5/17 μΌμš”μΌ λ¨Έμ§€ μ˜ˆμ •) +- 의쑴: βœ… PR #44 (#24 Opik 1단계) +- 후속: #18 Text2Cypher (Layer A 적재된 ν’μ„±ν•œ κ·Έλž˜ν”„ μœ„μ—μ„œ) +- 미래: #46 OpenAI λ§ˆμ΄κ·Έλ ˆμ΄μ…˜, #47 νšŒμ‚¬ λ°œν‘œ (6/1), [μ‹ κ·œ] Entity μΆ”μΆœ ν”„λ‘¬ν”„νŠΈ κ°œμ„  diff --git a/ingestion/chunker.py b/ingestion/chunker.py index 01ecd8f..e1733bb 100644 --- a/ingestion/chunker.py +++ b/ingestion/chunker.py @@ -1,14 +1,24 @@ # Ported from doc-summary-agent/chunker/chunker.py (#11). # Source: TaskerJang/doc-summary-agent@dev as of 2026-05-03. -# λ³€κ²½: μ—†μŒ (원본 κ·ΈλŒ€λ‘œ). νšŒμ‚¬ 레포의 인제슀트 κ²°κ³Όλ₯Ό λ™μΌν•˜κ²Œ μž¬ν˜„ν•˜κΈ° μœ„ν•œ -# μ˜λ„μ  볡뢙이며, GraphRAG vs VectorRAG λΉ„κ΅μ˜ 정직성을 μœ„ν•΄ chunk μ •μ±… -# (size=700, overlap=200, min=50)을 νšŒμ‚¬ 레포 dev와 λ™μΌν•˜κ²Œ μœ μ§€ν•œλ‹€. +# +# λ³€κ²½ 이λ ₯: +# - 5/16, #17: USE_SIMPLE_CHUNKER ν™˜κ²½λ³€μˆ˜ λΆ„κΈ° μΆ”κ°€ (CPU ν™˜κ²½ fallback). +# κΈ°λ³Έκ°’ OFF β€” ν™˜κ²½λ³€μˆ˜ μ•ˆ 켜면 νšŒμ‚¬ λ ˆν¬μ™€ 100% 동일 λ™μž‘. +# ON μ‹œ RecursiveCharacterTextSplitter μ‚¬μš© (SemanticChunker 우회). +# +# 정직성 원칙: +# chunk μ •μ±… (size=700, overlap=200, min=50) 은 νšŒμ‚¬ 레포 dev와 동일 μœ μ§€ β€” +# GraphRAG vs VectorRAG 비ꡐ μ‹œ chunk 경계 λ³€μˆ˜λ₯Ό ν†΅μ œν•˜κΈ° μœ„ν•΄. +# USE_SIMPLE_CHUNKER=1 fallback 은 CPU ν™˜κ²½μ—μ„œ SemanticChunker κ°€ λΉ„ν˜„μ‹€μ μΈ +# 경우 (820자/31초, 5/16 발견) 의 escape hatch β€” λ°œν‘œ 자료의 trade-off μŠ¬λΌμ΄λ“œ +# 증거둜 μ‚¬μš©. import logging +import os import re from collections import Counter from typing import Literal, TypedDict -from langchain_text_splitters import MarkdownTextSplitter +from langchain_text_splitters import MarkdownTextSplitter, RecursiveCharacterTextSplitter logger = logging.getLogger(__name__) @@ -46,6 +56,22 @@ # S-02: SemanticChunker μ‹±κΈ€ν„΄ _semantic_splitter = None +# ── #17 SIMPLE_CHUNKER fallback ──────────────────────────── +# +# USE_SIMPLE_CHUNKER=1 ν™˜κ²½λ³€μˆ˜ ON μ‹œ SemanticChunker μš°νšŒν•˜κ³  +# RecursiveCharacterTextSplitter μ‚¬μš©. +# +# λ°°κ²½ (5/16, #17): +# - LangChain SemanticChunker κ°€ μž„λ² λ”©μ„ λ¬Έμž₯ λ‹¨μœ„ sequential 호좜 (batch λ―Έμ‚¬μš©) +# - bge-m3 (568M params) + CPU ν™˜κ²½: 820자 / 31.3초 μΈ‘μ • (5/16 단독 벀치) +# - 8λ¬Έμ„œ ν’€ μ‹€ν–‰ μ˜ˆμƒ: λΆ„ν• λ§Œ 40λΆ„~1μ‹œκ°„ β†’ production λΉ„ν˜„μ‹€μ  +# - SemanticChunker λŠ” OpenAI API 같은 λΉ λ₯Έ μž„λ² λ”©κ³Όλ§Œ ν˜Έν™˜ (GPU/API ν™˜κ²½ μ „μ œ) +# +# κΈ°λ³Έκ°’ OFF β€” νšŒμ‚¬ 레포 동일 λ™μž‘ μœ μ§€. +# ON 으둜 ν‚€λ©΄ chunk 경계 μ•Œκ³ λ¦¬μ¦˜μ΄ νšŒμ‚¬ λ ˆν¬μ™€ λ‹¬λΌμ§€λ―€λ‘œ +# GraphRAG vs VectorRAG μ •λ°€ 비ꡐ μ‹œμ—λŠ” 끄고 μ‚¬μš©. +_SIMPLE_CHUNKER_ENV = "USE_SIMPLE_CHUNKER" + # ── #75 메타데이터 enrichment μƒμˆ˜ ────────────────────────── # M-01: 연도 μΆ”μΆœ μ •κ·œμ‹ β€” 2010~2039 λ²”μœ„ (금육 λ³΄κ³ μ„œ μ‹€μš© λ²”μœ„) @@ -153,6 +179,21 @@ def _get_semantic_splitter(): def _semantic_split(text: str) -> list[str]: + # #17 (5/16) β€” USE_SIMPLE_CHUNKER=1 켜져 있으면 SemanticChunker 우회. + # λ°°κ²½: bge-m3 + CPU + SemanticChunker = 820자 / 31.3초 (λΉ„ν˜„μ‹€μ ). + # λ°œν‘œ 자료의 trade-off μŠ¬λΌμ΄λ“œ 증거 + production fallback. + if os.getenv(_SIMPLE_CHUNKER_ENV) == "1": + logger.info( + "%s=1 β†’ SemanticChunker 우회, RecursiveCharacterTextSplitter μ‚¬μš©", + _SIMPLE_CHUNKER_ENV, + ) + simple_splitter = RecursiveCharacterTextSplitter( + chunk_size=DEFAULT_CHUNK_SIZE, + chunk_overlap=DEFAULT_CHUNK_OVERLAP, + separators=["\n\n", "\n", ". ", "。", "? ", "! ", " ", ""], + ) + return simple_splitter.split_text(text) + sentences = [s for s in re.split(KOREAN_SENTENCE_SPLIT_REGEX, text) if s.strip()] if len(sentences) < 3: logger.debug("λ¬Έμž₯ 수 λΆ€μ‘±(%d) β†’ SemanticChunker 우회", len(sentences)) diff --git a/kg/builder.py b/kg/builder.py index 754229b..29d3be6 100644 --- a/kg/builder.py +++ b/kg/builder.py @@ -1,23 +1,27 @@ -"""Layer B 적재 β€” LinkingResult + Relations β†’ Neo4j MERGE. +"""Layer A/B 적재 β€” Neo4j MERGE 기반 idempotent ingest. -#13 (extractor) β†’ #14 (linking) 의 좜λ ₯을 λ°›μ•„ Neo4j Aura 에 Layer B λ₯Ό -μ μž¬ν•œλ‹€. Layer A (Document/Section/Chunk) λŠ” 5/14 λ³΄λ„ˆμŠ€μ—μ„œ. +#13 (extractor) β†’ #14 (linking) 의 좜λ ₯을 λ°›μ•„ Neo4j Aura 에 Layer B λ₯Ό, +#17 의 ingestion.Document 객체λ₯Ό λ°›μ•„ Layer A λ₯Ό μ μž¬ν•œλ‹€. 섀계 원칙: - **Idempotent**: MERGE 기반 β€” μž¬μ‹€ν–‰ν•΄λ„ 쀑볡 λ…Έλ“œ μ•ˆ 생김 -- **λ©€ν‹° 라벨**: λͺ¨λ“  Entity λ…Έλ“œμ— `:Entity` (μ „μ—­ κ²€μƒ‰μš©) + ꡬ체 νƒ€μž… - (`:Company` `:Metric` `:Recommendation` `:Risk` `:Outlook`) λ™μ‹œ λΆ€μ°© -- **λ…Έλ“œ ν‚€**: `group_id` (NED κ°€ λΆ€μ—¬ν•œ μ „μ—­ ID) β€” 같은 그룹이면 같은 λ…Έλ“œ -- **관계**: ExtractedRelation.source/target (청크 λ‚΄ local_id) λŠ” NED 단계에 - group_id 둜 λ§€ν•‘ν•˜λŠ” 보쑰 ν…Œμ΄λΈ” ν•„μš” β€” λ³Έ λͺ¨λ“ˆμ΄ μ±…μž„ +- **λ©€ν‹° 라벨**: Layer B Entity λ…Έλ“œλŠ” `:Entity` (μ „μ—­ 검색) + ꡬ체 νƒ€μž… λ™μ‹œ λΆ€μ°© +- **λ…Έλ“œ ν‚€**: Layer B λŠ” `group_id` (NED), Layer A λŠ” `id` (file hash + 청크 idx) +- **Layer 뢄리**: Layer A 적재 μ‹œ Layer B λ…Έλ“œ μ•ˆ λ§Œμ§€κ³ , κ·Έ λ°˜λŒ€λ„ 동일. + 연결은 별도 ν•¨μˆ˜ `link_chunks_to_entities` κ°€ μ±…μž„ β€” AGENTS.md 원칙 + "Layer μ±…μž„ μ„žμ§€ μ•ŠκΈ°" μ€€μˆ˜. #24 (Opik νŠΈλ ˆμ΄μ‹±): -- 곡개 μ§„μž…μ  `build_layer_b` 에 `@track` β€” span μž…μΆœλ ₯은 λ…Έλ“œ/관계 적재 수. -- κ°œλ³„ MERGE 호좜 (_merge_group / _merge_relation) 은 λΆ€μ°© μ•ˆ 함 β€” 8λ¬Έμ„œ - 적재 μ‹œ span 폭증 λ°©μ§€ (1λ¬Έμ„œ = μˆ˜μ‹­ 개 MERGE). +- 곡개 μ§„μž…μ  3개 λͺ¨λ‘ `@track`: build_layer_a / build_layer_b / link_chunks_to_entities +- κ°œλ³„ MERGE ν˜ΈμΆœμ€ λΆ€μ°© μ•ˆ 함 β€” 8λ¬Έμ„œ 적재 μ‹œ span 폭증 λ°©μ§€ -κ΄€λ ¨ 이슈: #15 (λ³Έ), #13/#14 (μž…λ ₯ 곡급), #16 (후속 β€” μ‹œκ°ν™”), - #24 (Opik νŠΈλ ˆμ΄μ‹±). +#17 (5/16 8λ¬Έμ„œ 일괄): +- Layer A 적재 β€” doc-ontology.md Β§3-Β§4 의 Cypher μŠ€ν‚€λ§ˆ κ·ΈλŒ€λ‘œ κ΅¬ν˜„ +- 청크 λ…Έλ“œμ˜ id κ°€ ingestion.adapter._chunk_id() 의 `{doc_id}:c{idx:04d}` ν˜•μ‹ +- entity 의 local_id λŠ” extractor μ˜΅μ…˜ 3 으둜 `{chunk_id}__ent_001` ν˜•μ‹ β†’ + parse_global_id 둜 청크 역좔적해 MENTIONS 관계 생성 + +κ΄€λ ¨ 이슈: #15 (Layer B), #17 (Layer A + 8λ¬Έμ„œ), #13/#14 (μž…λ ₯), #24 (Opik). """ from __future__ import annotations @@ -26,15 +30,22 @@ import time from dataclasses import dataclass +from kg.extractor import parse_global_id from kg.linking import EntityGroup, LinkingResult from kg.neo4j_client import Neo4jClient from kg.ontology import EntityType, ExtractedEntity, ExtractedRelation, RelationType from observability.tracing import track +# Layer A λͺ¨λΈ β€” ingestion λͺ¨λ“ˆμ—μ„œ import. circular λ°©μ§€λ₯Ό μœ„ν•΄ ν•¨μˆ˜ μ•ˆμ—μ„œ +# import ν•˜μ§€ μ•Šκ³  μ—¬κΈ°μ„œ ν•œ 번만. +from ingestion.models import Chunk, ChunkType, Document, Section, Table + logger = logging.getLogger(__name__) -# ── κ²°κ³Ό νƒ€μž… ─────────────────────────────────────────── +# ═══════════════════════════════════════════════════════════════════ +# κ²°κ³Ό νƒ€μž… +# ═══════════════════════════════════════════════════════════════════ @dataclass class IngestResult: """build_layer_b() 의 적재 κ²°κ³Ό 수치.""" @@ -45,13 +56,44 @@ class IngestResult: elapsed_sec: float -# ── 인덱슀 / μ œμ•½ 쑰건 ─────────────────────────────────── +@dataclass +class LayerAIngestResult: + """build_layer_a() 의 적재 κ²°κ³Ό 수치.""" + + documents_merged: int + sections_merged: int + chunks_merged: int + tables_merged: int + next_relations: int # Chunk -[:NEXT]-> Chunk 개수 + elapsed_sec: float + + +@dataclass +class MentionsResult: + """link_chunks_to_entities() 의 적재 κ²°κ³Ό 수치.""" + + mentions_merged: int + mentions_dropped: int # chunk_id parse μ‹€νŒ¨ λ˜λŠ” entity μ—†μŒ + elapsed_sec: float + + +# ═══════════════════════════════════════════════════════════════════ +# 인덱슀 / μ œμ•½ 쑰건 +# ═══════════════════════════════════════════════════════════════════ INDEX_QUERIES = [ - # group_id 유일 μ œμ•½ β€” λ™μ‹œμ— B-tree 인덱슀 μ—­ν•  + # Layer B "CREATE CONSTRAINT entity_group_id IF NOT EXISTS " "FOR (e:Entity) REQUIRE e.group_id IS UNIQUE", - # name 검색 가속 (유일 μ œμ•½ μ•„λ‹˜ β€” 동λͺ… νšŒμ‚¬Β·λ©”νŠΈλ¦­ κ°€λŠ₯) "CREATE INDEX entity_name IF NOT EXISTS FOR (e:Entity) ON (e.name)", + # Layer A β€” #17 μΆ”κ°€. doc-ontology.md Β§3 의 4 λ…Έλ“œ λͺ¨λ‘ id 유일. + "CREATE CONSTRAINT document_id IF NOT EXISTS " + "FOR (d:Document) REQUIRE d.id IS UNIQUE", + "CREATE CONSTRAINT section_id IF NOT EXISTS " + "FOR (s:Section) REQUIRE s.id IS UNIQUE", + "CREATE CONSTRAINT chunk_id IF NOT EXISTS " + "FOR (c:Chunk) REQUIRE c.id IS UNIQUE", + "CREATE CONSTRAINT table_id IF NOT EXISTS " + "FOR (t:Table) REQUIRE t.id IS UNIQUE", ] @@ -62,7 +104,9 @@ def ensure_indexes(client: Neo4jClient) -> None: logger.info("Index/constraint applied: %s", q.split("FOR")[0].strip()) -# ── λ…Έλ“œ MERGE ────────────────────────────────────────── +# ═══════════════════════════════════════════════════════════════════ +# Layer B β€” Entity λ…Έλ“œ / 관계 +# ═══════════════════════════════════════════════════════════════════ # λ©€ν‹° 라벨 β€” EntityType 값을 동적 라벨둜 λΆ€μ°©ν•˜λ©΄μ„œλ„ 곡톡 :Entity 도 λΆ€μ—¬. # Cypher λŠ” 라벨을 νŒŒλΌλ―Έν„°λ‘œ 받을 수 μ—†μœΌλ―€λ‘œ f-string 으둜 쑰립 β€” # μž…λ ₯ EntityType.value λŠ” enum μ—μ„œ 온 ν™”μ΄νŠΈλ¦¬μŠ€νŠΈ (injection μ—†μŒ). @@ -91,7 +135,6 @@ def _merge_group(client: Neo4jClient, group: EntityGroup) -> None: ) -# ── 관계 MERGE ───────────────────────────────────────── RELATION_MERGE_TEMPLATE = """ MATCH (s:Entity {{group_id: $source_gid}}) MATCH (t:Entity {{group_id: $target_gid}}) @@ -118,16 +161,11 @@ def _merge_relation( ) -# ── local_id β†’ group_id λ§€ν•‘ ────────────────────────────────── def _build_local_to_group(linking: LinkingResult) -> dict[str, str]: """ExtractedEntity.local_id β†’ 그게 μ†ν•œ EntityGroup.group_id. - ExtractedRelation 은 source/target 으둜 local_id λ₯Ό μ‹¨λŠ”λ° (μΆ”μΆœ λ‹Ήμ‹œμ˜ ID), - 적재 μ‹œμ μ—λŠ” NED κ·Έλ£Ή κΈ°μ€€μœΌλ‘œ μ—°κ²°ν•΄μ•Ό ν•˜λ―€λ‘œ λ§€ν•‘ ν•„μš”. - - 주의: local_id λŠ” 청크 κ°„ 쀑볡 κ°€λŠ₯ (각 청크가 ent_001 λΆ€ν„° μ‹œμž‘). - μ‹€μ œ μš΄μ˜μ—μ„œλŠ” (chunk_idx, local_id) νŠœν”Œμ΄ key μ—¬μ•Ό μ•ˆμ „. - λ³Έ PR 은 1청크 μ‹œλ‚˜λ¦¬μ˜€λ§Œ μ·¨κΈ‰ β€” #17 μ—μ„œ 닀쀑 청크 μ‹œ κ²°ν•©. + μ˜΅μ…˜ 3 (#17) 이후 local_id λŠ” `{chunk_id}__ent_001` ν˜•μ‹ β€” global 유일. + λ”°λΌμ„œ 청크 κ°„ 좩돌 μ—†μŒ. """ mapping: dict[str, str] = {} for group in linking.groups: @@ -136,7 +174,6 @@ def _build_local_to_group(linking: LinkingResult) -> dict[str, str]: return mapping -# ── 곡개 μΈν„°νŽ˜μ΄μŠ€ ─────────────────────────────────────── @track def build_layer_b( linking: LinkingResult, @@ -151,38 +188,22 @@ def build_layer_b( 1. (μ˜΅μ…˜) 인덱슀/μ œμ•½μ‘°κ±΄ λ©±λ“± 생성 2. κ·Έλ£Ήλ³„λ‘œ :Entity:{Type} λ…Έλ“œ MERGE 3. local_id β†’ group_id λ§€ν•‘ ꡬ좕 - 4. κ΄€κ³„λ³„λ‘œ MATCH-MATCH-MERGE (μ–‘μͺ½ κ·Έλ£Ή 쑴재 ν•„μˆ˜) - - Args: - linking: #14 좜λ ₯ (κ·Έλ£Ήν™”λœ Entity λ“€) - relations: #13 좜λ ₯ (청크 λ‚΄ local_id 기반 관계) - client: Neo4jClient λ₯Ό μ£Όμž… (ν…ŒμŠ€νŠΈΒ·CLI μž¬ν™œμš©). None 이면 from_env. - create_indexes: 첨 적재 μ‹œ 인덱슀 생성 μ‹œλ„ (λ©±λ“±). - - Returns: - IngestResult β€” 적재 수치. - - #24 Opik: - - λ³Έ ν•¨μˆ˜μ— `@track` β€” span μž…μΆœλ ₯은 λ…Έλ“œ/관계 적재 μˆ˜Β·μ†Œμš”. - - ν™˜κ²½ λ―Έμ„€μ • μ‹œ μžλ™ no-op. + 4. κ΄€κ³„λ³„λ‘œ MATCH-MATCH-MERGE """ started = time.perf_counter() if client is None: - client = Neo4jClient() # from_env + client = Neo4jClient() if create_indexes: ensure_indexes(client) - # 2. λ…Έλ“œ MERGE for group in linking.groups: _merge_group(client, group) nodes_merged = len(linking.groups) - # 3. local_id β†’ group_id local_to_group = _build_local_to_group(linking) - # 4. 관계 MERGE relations_merged = 0 relations_dropped = 0 for rel in relations: @@ -210,3 +231,323 @@ def build_layer_b( result.nodes_merged, result.relations_merged, result.relations_dropped, result.elapsed_sec, ) return result + + +# ═══════════════════════════════════════════════════════════════════ +# Layer A β€” Document / Section / Chunk / Table (#17) +# ═══════════════════════════════════════════════════════════════════ +# λ…Έλ“œ MERGE ν…œν”Œλ¦Ώ. doc-ontology.md Β§3.1-3.4 μŠ€ν‚€λ§ˆ κ·ΈλŒ€λ‘œ. + +DOCUMENT_MERGE = """ +MERGE (d:Document {id: $id}) +SET d.filename = $filename, + d.doc_type = $doc_type, + d.source_format = $source_format, + d.total_pages = $total_pages, + d.publisher = $publisher, + d.subject = $subject, + d.fiscal_year = $fiscal_year, + d.ingested_at = timestamp() +RETURN d.id AS id +""".strip() + + +SECTION_MERGE = """ +MERGE (s:Section {id: $id}) +SET s.label = $label, + s.doc_type = $doc_type, + s.original_section = $original_section, + s.heading_level = $heading_level, + s.order_index = $order_index, + s.page_start = $page_start, + s.page_end = $page_end, + s.updated_at = timestamp() +RETURN s.id AS id +""".strip() + + +CHUNK_MERGE = """ +MERGE (c:Chunk {id: $id}) +SET c.text = $text, + c.chunk_type = $chunk_type, + c.char_count = $char_count, + c.page = $page, + c.order_index = $order_index, + c.updated_at = timestamp() +RETURN c.id AS id +""".strip() + + +TABLE_MERGE = """ +MERGE (t:Table {id: $id}) +SET t.raw_markdown = $raw_markdown, + t.caption = $caption, + t.row_count = $row_count, + t.column_count = $column_count, + t.page = $page, + t.order_index = $order_index, + t.updated_at = timestamp() +RETURN t.id AS id +""".strip() + + +# 관계 MERGE β€” Layer A 의 4 관계 (doc-ontology.md Β§4) +HAS_SECTION_MERGE = """ +MATCH (d:Document {id: $doc_id}) +MATCH (s:Section {id: $section_id}) +MERGE (d)-[:HAS_SECTION]->(s) +""".strip() + + +CONTAINS_CHUNK_MERGE = """ +MATCH (s:Section {id: $section_id}) +MATCH (c:Chunk {id: $chunk_id}) +MERGE (s)-[:CONTAINS_CHUNK]->(c) +""".strip() + + +CONTAINS_TABLE_MERGE = """ +MATCH (s:Section {id: $section_id}) +MATCH (t:Table {id: $table_id}) +MERGE (s)-[:CONTAINS_TABLE]->(t) +""".strip() + + +NEXT_CHUNK_MERGE = """ +MATCH (a:Chunk {id: $prev_id}) +MATCH (b:Chunk {id: $next_id}) +MERGE (a)-[:NEXT]->(b) +""".strip() + + +def _merge_document(client: Neo4jClient, doc: Document) -> None: + """Document λ…Έλ“œ MERGE.""" + client.write( + DOCUMENT_MERGE, + id=doc.id, + filename=doc.filename, + doc_type=doc.doc_type.value, + source_format=doc.source_format.value, + total_pages=doc.total_pages, + publisher=doc.publisher, + subject=doc.subject, + fiscal_year=doc.fiscal_year, + ) + + +def _merge_section(client: Neo4jClient, section: Section) -> None: + """Section λ…Έλ“œ MERGE β€” label 은 W3 LLM 채움 전이라 None κ°€λŠ₯.""" + client.write( + SECTION_MERGE, + id=section.id, + label=section.label.value if section.label else None, + doc_type=section.doc_type.value, + original_section=section.original_section, + heading_level=section.heading_level, + order_index=section.order_index, + page_start=section.page_start, + page_end=section.page_end, + ) + + +def _merge_chunk(client: Neo4jClient, chunk: Chunk) -> None: + """Chunk λ…Έλ“œ MERGE.""" + client.write( + CHUNK_MERGE, + id=chunk.id, + text=chunk.text, + chunk_type=chunk.chunk_type.value, + char_count=chunk.char_count, + page=chunk.page, + order_index=chunk.order_index, + ) + + +def _merge_table(client: Neo4jClient, table: Table) -> None: + """Table λ…Έλ“œ MERGE.""" + client.write( + TABLE_MERGE, + id=table.id, + raw_markdown=table.raw_markdown, + caption=table.caption, + row_count=table.row_count, + column_count=table.column_count, + page=table.page, + order_index=table.order_index, + ) + + +@track(ignore_arguments=["client"]) +def build_layer_a( + document: Document, + *, + client: Neo4jClient | None = None, + create_indexes: bool = True, +) -> LayerAIngestResult: + """Layer A (Document Structure) λ₯Ό Neo4j 에 적재. + + νŒŒμ΄ν”„λΌμΈ (doc-ontology.md Β§3-Β§4): + 1. (μ˜΅μ…˜) Layer A 인덱슀/μ œμ•½μ‘°κ±΄ λ©±λ“± 생성 + 2. Document λ…Έλ“œ MERGE + 3. Section λ…Έλ“œλ“€ MERGE + HAS_SECTION 관계 + 4. Chunk / Table λ…Έλ“œλ“€ MERGE + CONTAINS_CHUNK / CONTAINS_TABLE 관계 + 5. Section λ‚΄ Chunk μˆœμ„œλŒ€λ‘œ NEXT 관계 + + Args: + document: ingestion.adapter.parse_document() 좜λ ₯ + client: Neo4jClient μ£Όμž… (ν…ŒμŠ€νŠΈ μž¬ν™œμš©). None 이면 from_env. + create_indexes: 첫 적재 μ‹œ 인덱슀 생성 μ‹œλ„ (λ©±λ“±). + + Returns: + LayerAIngestResult β€” λ…Έλ“œ/관계 적재 수치. + + #24 Opik: + - λ³Έ ν•¨μˆ˜μ— `@track(ignore_arguments=["client"])` β€” client 객체가 + `<...object at 0x...>` 둜 λ°•ν˜€ λ…Έμ΄μ¦ˆκ°€ λ˜λŠ” 것 νšŒν”Ό (5/16 λ°•μ œ). + """ + started = time.perf_counter() + + if client is None: + client = Neo4jClient() + + if create_indexes: + ensure_indexes(client) + + # 2. Document + _merge_document(client, document) + documents_merged = 1 + + sections_merged = 0 + chunks_merged = 0 + tables_merged = 0 + next_relations = 0 + + # 3-5. Section / Chunk / Table / 관계 + for section in document.sections: + _merge_section(client, section) + client.write(HAS_SECTION_MERGE, doc_id=document.id, section_id=section.id) + sections_merged += 1 + + prev_chunk_id: str | None = None + for chunk in section.chunks: + _merge_chunk(client, chunk) + client.write( + CONTAINS_CHUNK_MERGE, + section_id=section.id, + chunk_id=chunk.id, + ) + chunks_merged += 1 + + # Section λ‚΄ Chunk NEXT 관계 + if prev_chunk_id is not None: + client.write( + NEXT_CHUNK_MERGE, + prev_id=prev_chunk_id, + next_id=chunk.id, + ) + next_relations += 1 + prev_chunk_id = chunk.id + + for table in section.tables: + _merge_table(client, table) + client.write( + CONTAINS_TABLE_MERGE, + section_id=section.id, + table_id=table.id, + ) + tables_merged += 1 + + elapsed = time.perf_counter() - started + result = LayerAIngestResult( + documents_merged=documents_merged, + sections_merged=sections_merged, + chunks_merged=chunks_merged, + tables_merged=tables_merged, + next_relations=next_relations, + elapsed_sec=elapsed, + ) + logger.info( + "Layer A 적재 μ™„λ£Œ doc=%s sections=%d chunks=%d tables=%d next=%d (%.2fs)", + document.filename, + result.sections_merged, result.chunks_merged, + result.tables_merged, result.next_relations, + result.elapsed_sec, + ) + return result + + +# ═══════════════════════════════════════════════════════════════════ +# MENTIONS β€” Layer A Chunk β†’ Layer B Entity (#17) +# ═══════════════════════════════════════════════════════════════════ +MENTIONS_MERGE = """ +MATCH (c:Chunk {id: $chunk_id}) +MATCH (e:Entity {group_id: $group_id}) +MERGE (c)-[:MENTIONS]->(e) +""".strip() + + +@track(ignore_arguments=["client"]) +def link_chunks_to_entities( + linking: LinkingResult, + *, + client: Neo4jClient | None = None, +) -> MentionsResult: + """Layer A Chunk 와 Layer B Entity λ₯Ό MENTIONS κ΄€κ³„λ‘œ μ—°κ²°. + + λ™μž‘: + - linking.groups 의 각 EntityGroup.members λ₯Ό 순회 + - 각 member 의 local_id μ—μ„œ `parse_global_id` 둜 chunk_id μΆ”μΆœ + - `(:Chunk {id: chunk_id}) -[:MENTIONS]-> (:Entity {group_id: group.group_id})` + + chunk_id κ°€ μ—†λŠ” (ꡬ식, 5/10 sanity) entity λŠ” graceful drop β€” Layer A + κ°€ 적재 μ•ˆ 됐을 κ°€λŠ₯μ„± λ†’μœΌλ―€λ‘œ λ§€ν•‘ λΆˆκ°€. + + Args: + linking: #14 좜λ ₯ (κ·Έλ£Ήν™”λœ Entity λ“€). 각 member 의 local_id κ°€ + `{chunk_id}__ent_001` ν˜•μ‹μ΄μ–΄μ•Ό λ§€ν•‘ κ°€λŠ₯ (μ˜΅μ…˜ 3). + client: Neo4jClient μ£Όμž…. None 이면 from_env. + + Returns: + MentionsResult β€” 적재 / drop 수치. + """ + started = time.perf_counter() + + if client is None: + client = Neo4jClient() + + mentions_merged = 0 + mentions_dropped = 0 + + for group in linking.groups: + # 같은 κ·Έλ£Ή μ•ˆμ—μ„œ 동일 청크가 μ—¬λŸ¬ member 둜 λ“±μž₯ν•  수 있음 (λ“œλ¬Όκ²Œ). + # MERGE κ°€ idempotent 라 μ—¬λŸ¬ 번 ν˜ΈμΆœν•΄λ„ μ•ˆμ „ν•˜μ§€λ§Œ ν†΅κ³„λŠ” μ •ν™•νžˆ + # 청크별 1회둜 집계. + seen_chunks: set[str] = set() + for member in group.members: + chunk_id, _ = parse_global_id(member.local_id) + if chunk_id is None: + # ꡬ식 local_id β€” Layer A λ§€ν•‘ λΆˆκ°€. 5/10 sanity ν˜Έν™˜. + mentions_dropped += 1 + continue + if chunk_id in seen_chunks: + continue + seen_chunks.add(chunk_id) + + client.write( + MENTIONS_MERGE, + chunk_id=chunk_id, + group_id=group.group_id, + ) + mentions_merged += 1 + + elapsed = time.perf_counter() - started + result = MentionsResult( + mentions_merged=mentions_merged, + mentions_dropped=mentions_dropped, + elapsed_sec=elapsed, + ) + logger.info( + "MENTIONS 적재 μ™„λ£Œ merged=%d dropped=%d (%.2fs)", + result.mentions_merged, result.mentions_dropped, result.elapsed_sec, + ) + return result diff --git a/kg/extractor.py b/kg/extractor.py index 39747e6..4b9e098 100644 --- a/kg/extractor.py +++ b/kg/extractor.py @@ -18,8 +18,15 @@ - 청크 λ‹¨μœ„ (_extract_chunk) λŠ” μ˜λ„μ μœΌλ‘œ λΆ€μ°© μ•ˆ 함 β€” 8λ¬Έμ„œ Γ— N청크 일괄 적재 μ‹œ trace κ°€ ν­λ°œν•˜μ§€ μ•Šλ„λ‘. ν•„μš”ν•˜λ©΄ W5 μ—μ„œ μΆ”κ°€. +#17 (5/16 8λ¬Έμ„œ 일괄 적재) β€” local_id 좩돌 ν•΄κ²°: +- 청크 N κ°œμ—μ„œ 각각 `ent_001` λΆ€ν„° λΆ€μ—¬ β†’ 닀쀑 청크 μ‹œ local_id 좩돌 +- ν•΄κ²° (μ˜΅μ…˜ 3): chunk dict 에 μ˜΅μ…”λ„ `chunk_id` ν‚€λ₯Ό μΆ”κ°€ν•˜λ©΄ λ³Έ λͺ¨λ“ˆμ΄ + μžλ™μœΌλ‘œ λͺ¨λ“  entity local_id 와 relation source/target 에 + `{chunk_id}__{local_id}` prefix λΆ€μ—¬ β€” global μœ μΌμ„± 확보. +- chunk_id μ—†λŠ” 호좜 (5/10 두산λ°₯μΊ£ 1청크 sanity) 은 κΈ°μ‘΄ λ™μž‘ κ·ΈλŒ€λ‘œ. + κ΄€λ ¨ 이슈: #13 (λ³Έ μž‘μ—…), #8 (Spike), #14 (NED β€” λ³Έ λͺ¨λ“ˆμ˜ 좜λ ₯이 μž…λ ₯), - #24 (Opik νŠΈλ ˆμ΄μ‹±). + #17 (8λ¬Έμ„œ 일괄), #24 (Opik νŠΈλ ˆμ΄μ‹±). """ from __future__ import annotations @@ -68,6 +75,11 @@ SYSTEM_PROMPT_PATH = PROMPTS_DIR / "entity_system_v1.md" EXTRACT_PROMPT_PATH = PROMPTS_DIR / "entity_extract_v1.md" +# local_id global prefix κ΅¬λΆ„μž. +# `{chunk_id}__{local_id}` 의 `__` β€” '_' 단일은 chunk_id μ•ˆμ— ν”νžˆ λ“±μž₯ (예: +# `doc_abc:c0001`) ν•˜λ―€λ‘œ 좩돌 νšŒν”Ό μœ„ν•΄ 이쀑 underscore. +_GLOBAL_ID_SEP = "__" + # ── λͺ¨λ“ˆ 레벨 Semaphore (이벀트 루프 생λͺ…μ£ΌκΈ° 곡유) ────────── _sem: asyncio.Semaphore | None = None @@ -118,6 +130,66 @@ def _parse_extraction_json(raw: str) -> dict[str, Any]: return json.loads(fixed) # 두 λ²ˆμ§Έλ„ μ‹€νŒ¨ν•˜λ©΄ ν˜ΈμΆœμžκ°€ 작음 +# ── chunk_id prefix λΆ€μ—¬ (μ˜΅μ…˜ 3 β€” #17) ────────────────────── +def make_global_id(chunk_id: str, local_id: str) -> str: + """`{chunk_id}__{local_id}` ν˜•μ‹μ˜ global id 생성. + + μ—­λ°©ν–₯ νŒŒμ‹± (`parse_global_id`) 으둜 μ–΄λŠ μ²­ν¬μ—μ„œ λ‚˜μ˜¨ entity 인지 + 좔적 κ°€λŠ₯ β€” Layer A 의 `(:Chunk) -[:MENTIONS]-> (:Entity)` 맀핑에 μ‚¬μš©. + """ + return f"{chunk_id}{_GLOBAL_ID_SEP}{local_id}" + + +def parse_global_id(global_id: str) -> tuple[str | None, str]: + """global id λ₯Ό (chunk_id, local_id) 둜 λΆ„ν•΄. + + prefix κ°€ μ—†λŠ” (ꡬ식) id λŠ” (None, id) 둜 λ°˜ν™˜ β€” 5/10 두산λ°₯μΊ£ 1청크 + sanity 와 ν˜Έν™˜. + """ + if _GLOBAL_ID_SEP in global_id: + chunk_id, local_id = global_id.split(_GLOBAL_ID_SEP, 1) + return chunk_id, local_id + return None, global_id + + +def _apply_chunk_id_prefix( + result: ExtractionResult, chunk_id: str +) -> ExtractionResult: + """ExtractionResult 의 λͺ¨λ“  local_id / source / target 에 chunk_id prefix. + + `_coerce_to_result` κ°€ source/target 의 local_id μœ νš¨μ„± 검증을 마친 ν›„ + ν˜ΈμΆœλ˜λ―€λ‘œ, λ³Έ ν•¨μˆ˜λŠ” λ§€ν•‘λ§Œ μ•ˆμ „ν•˜κ²Œ μˆ˜ν–‰. + """ + if not chunk_id: + return result + + # entity: local_id 만 prefix λΆ€μ—¬ + new_entities = [ + ExtractedEntity( + local_id=make_global_id(chunk_id, e.local_id), + type=e.type, + canonical=e.canonical, + source_span=e.source_span, + section=e.section, + ) + for e in result.entities + ] + + # relation: source / target λ‘˜ λ‹€ prefix (같은 청크 λ‚΄ κ΄€κ³„λ§Œ μΆ”μΆœλ˜λ―€λ‘œ + # 동일 chunk_id 둜 prefix). + new_relations = [ + ExtractedRelation( + source=make_global_id(chunk_id, r.source), + type=r.type, + target=make_global_id(chunk_id, r.target), + evidence=r.evidence, + ) + for r in result.relations + ] + + return ExtractionResult(entities=new_entities, relations=new_relations) + + # ── 응닡 β†’ Pydantic λ³€ν™˜ ───────────────────────────────────── def _coerce_to_result(data: dict[str, Any], section: str | None) -> ExtractionResult: """JSON dict β†’ ExtractionResult. @@ -214,12 +286,14 @@ async def _extract_chunk( ) -> ExtractionResult: """Chunk β†’ ExtractionResult. - chunk dict μ˜ˆμƒ ν‚€: doc_id, section, text. κ²°μΈ‘ μ‹œ placeholder 둜 graceful. - Semaphore 둜 λ™μ‹œμ„± μ œν•œ + asyncio.to_thread 둜 sync LLMClient 감쌈. + chunk dict μ˜ˆμƒ ν‚€: doc_id, section, text, (optional) chunk_id. + κ²°μΈ‘ μ‹œ placeholder 둜 graceful. + chunk_id κ°€ 있으면 결과의 local_id 듀에 μžλ™μœΌλ‘œ prefix λΆ€μ—¬ (#17). """ - doc_id = chunk.get("doc_id", "(unknown)") - section = chunk.get("section") or "(no section)" - text = (chunk.get("text") or "").strip() + doc_id = chunk.get("doc_id", "(unknown)") + section = chunk.get("section") or "(no section)" + text = (chunk.get("text") or "").strip() + chunk_id = chunk.get("chunk_id") # #17 β€” global id prefix 용 (μ—†μœΌλ©΄ prefix μ•ˆ 함) if not text: logger.info("빈 청크 skip (doc=%s section=%s)", doc_id, section) @@ -251,9 +325,15 @@ async def _extract_chunk( return ExtractionResult() result = _coerce_to_result(data, section=section) + + # #17 β€” chunk_id κ°€ μ£Όμ–΄μ§€λ©΄ global id prefix. μ—†μœΌλ©΄ (5/10 sanity) 원본 κ·ΈλŒ€λ‘œ. + if chunk_id: + result = _apply_chunk_id_prefix(result, chunk_id) + logger.info( - "μΆ”μΆœ OK doc=%s section=%s entities=%d relations=%d (%.2fs)", - doc_id, section, len(result.entities), len(result.relations), elapsed, + "μΆ”μΆœ OK doc=%s chunk=%s section=%s entities=%d relations=%d (%.2fs)", + doc_id, chunk_id or "(no-id)", section, + len(result.entities), len(result.relations), elapsed, ) return result @@ -268,9 +348,10 @@ async def extract( - μ²­ν¬λ³„λ‘œ 병렬 μΆ”μΆœ (Semaphore μ œν•œ) - κ²°κ³ΌλŠ” entities/relations λͺ¨λ‘ ν‰νƒ„ν™”ν•˜μ—¬ ν•©μΉ¨ - - local_id λŠ” 청크 λ‚΄μ—μ„œλ§Œ 유효 β†’ 호좜자(NED #14)κ°€ 청크 경계 μ•Œμ•„μ•Ό 함 - λŒ€μ•ˆ: prefix λΆ€μ—¬ (`{chunk_idx}__{local_id}`) β€” λ³Έ μž‘μ—…μ€ λ‹¨μˆœ ν•©μΉ¨μœΌλ‘œ, - NED λ‹¨κ³„μ—μ„œ 청크 경계 처리. + - **chunk dict 에 μ˜΅μ…”λ„ `chunk_id` ν‚€κ°€ 있으면** λ³Έ λͺ¨λ“ˆμ΄ μžλ™μœΌλ‘œ + λͺ¨λ“  entity local_id 와 relation source/target 에 + `{chunk_id}__{local_id}` prefix λΆ€μ—¬ β†’ 청크 κ°„ global μœ μΌμ„± (#17). + μ—†μœΌλ©΄ (5/10 sanity ν˜Έν™˜) 원본 local_id μœ μ§€. DoD (#13): - 빈 청크 / 인식 μ‹€νŒ¨ 청크 graceful (KeyError λ“± X) @@ -288,9 +369,11 @@ async def extract( extract_tmpl = _load_prompt(EXTRACT_PROMPT_PATH) started = time.perf_counter() + has_chunk_id = any(c.get("chunk_id") for c in chunks) logger.info( - "Entity μΆ”μΆœ μ‹œμž‘ β€” 청크 %d개 MAX_CONCURRENT=%d model=%s", + "Entity μΆ”μΆœ μ‹œμž‘ β€” 청크 %d개 MAX_CONCURRENT=%d model=%s global_id=%s", len(chunks), MAX_CONCURRENT, llm.config.model, + "ON" if has_chunk_id else "off", ) results: list[ExtractionResult] = await asyncio.gather( diff --git a/scripts/run_w3_batch.py b/scripts/run_w3_batch.py new file mode 100644 index 0000000..cc52d45 --- /dev/null +++ b/scripts/run_w3_batch.py @@ -0,0 +1,258 @@ +"""W3 batch β€” 평가 μ…‹ 8λ¬Έμ„œ 일괄 적재. + +`eval/dataset/documents/` 의 λͺ¨λ“  νŒŒμΌμ„ μˆœνšŒν•˜λ©° ν’€ νŒŒμ΄ν”„λΌμΈ: + parse_document (Layer A ꡬ쑰화) β†’ extract (Kimi) β†’ link_entities (bge-m3) + β†’ build_layer_a β†’ build_layer_b β†’ link_chunks_to_entities (MENTIONS) + +λ¬Έμ„œλ³„ 톡계 (청크 / Entity / Relation / μ†Œμš”μ‹œκ°„ / μ‹€νŒ¨ μ—¬λΆ€) μˆ˜μ§‘ ν›„ +λ§ˆμ§€λ§‰μ— ν‘œλ‘œ 좜λ ₯. weekly-log λ°•μ œ + 5/23 λ°œν‘œ 자료 Β§ 운영 데이터 μ‹œλ“œ. + +μ‚¬μš©: + uv run python -m scripts.run_w3_batch + uv run python -m scripts.run_w3_batch --limit 2 # 첫 2λ¬Έμ„œλ§Œ (sanity) + uv run python -m scripts.run_w3_batch --pattern "*.pdf" # PDF 만 + +μ „μ œμ‘°κ±΄: + .env 에 KIMI_API_KEY / NEO4J_URI / NEO4J_PASSWORD / OPIK_* + eval/dataset/documents/ 에 평가 μ…‹ 8λ¬Έμ„œ (gitignore β€” 둜컬만) + bge-m3 졜초 1회 λ‹€μš΄λ‘œλ“œ μ™„λ£Œ μƒνƒœ + +#17 μ‹œν–‰μ°©μ˜€ / 운영 데이터: +- λ³Έ 슀크립트 좜λ ₯ ν‘œλ₯Ό docs/weekly-log/2026-05-16-batch-ingest.md 에 λ°•μ œ +- Opik UI μ—μ„œ trace νŽ˜μ΄μ§€ 캑처 β†’ λ°œν‘œ Β§ How μŠ¬λΌμ΄λ“œ μ‹œλ“œ + +κ΄€λ ¨ 이슈: #17 (λ³Έ μž‘μ—…), #13 #14 #15 (1청크 검증 끝), #24 (Opik on). +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +import time +import traceback +from dataclasses import dataclass, field +from pathlib import Path + +from ingestion.adapter import parse_document +from ingestion.models import ChunkType, Document +from kg.builder import build_layer_a, build_layer_b, link_chunks_to_entities +from kg.extractor import extract +from kg.linking import link_entities +from kg.neo4j_client import Neo4jClient +from kg.ontology import ExtractionResult + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger("w3-batch") + +DOCS_DIR = Path("eval/dataset/documents") + + +@dataclass +class DocStat: + """λ¬Έμ„œλ³„ 처리 톡계 β€” λ§ˆμ§€λ§‰μ— ν‘œλ‘œ 좜λ ₯.""" + + filename: str + source_format: str + doc_type: str + success: bool = False + error: str | None = None + + sections: int = 0 + chunks_text: int = 0 + chunks_table: int = 0 + entities_raw: int = 0 + entities_grouped: int = 0 + relations: int = 0 + mentions: int = 0 + + sec_parse: float = 0.0 + sec_extract: float = 0.0 + sec_linking: float = 0.0 + sec_neo4j: float = 0.0 + sec_total: float = 0.0 + + +def _doc_to_extract_chunks(doc: Document) -> list[dict]: + """Layer A Document β†’ extract() κ°€ λ°›λŠ” chunk dict 리슀트. + + μ˜΅μ…˜ 3 (#17): chunk_id ν‚€λ₯Ό ν•¨κ»˜ λ°•μ•„ 보내 entity local_id 에 global + prefix κ°€ μžλ™ λΆ€μ—¬λ˜λ„λ‘. + """ + out: list[dict] = [] + for section in doc.sections: + for chunk in section.chunks: + # Table 은 별도 Layer A λ…Έλ“œ. extractor λŠ” text 청크만 λ°›λŠ” 게 λ‹¨μˆœ. + if chunk.chunk_type != ChunkType.TEXT: + continue + text = (chunk.text or "").strip() + if not text: + continue + out.append({ + "doc_id": doc.id, + "chunk_id": chunk.id, # ← #17 μ˜΅μ…˜ 3 핡심 ν‚€ + "section": section.original_section, + "text": text, + }) + return out + + +async def _process_one( + path: Path, + client: Neo4jClient, + *, + create_indexes: bool, +) -> DocStat: + """1 λ¬Έμ„œ ν’€ νŒŒμ΄ν”„λΌμΈ. μ‹€νŒ¨ν•΄λ„ μ˜ˆμ™Έ μ•ˆ λ˜μ§€κ³  stat 에 λ°•μ œ.""" + started_total = time.perf_counter() + + stat = DocStat( + filename=path.name, + source_format=path.suffix.lstrip(".").lower(), + doc_type="(unknown)", + ) + + try: + # 1. Parse β€” Layer A ꡬ쑰화 + t0 = time.perf_counter() + document = parse_document(path) + stat.sec_parse = time.perf_counter() - t0 + stat.doc_type = document.doc_type.value + stat.sections = len(document.sections) + for sec in document.sections: + stat.chunks_text += len(sec.chunks) + stat.chunks_table += len(sec.tables) + + logger.info( + "[%s] Parse OK sections=%d chunks(text)=%d tables=%d (%.2fs)", + path.name, stat.sections, stat.chunks_text, stat.chunks_table, stat.sec_parse, + ) + + # 2. Extract β€” Kimi + chunks = _doc_to_extract_chunks(document) + if not chunks: + logger.warning("[%s] text 청크 μ—†μŒ β€” extract skip", path.name) + ext_result = ExtractionResult() + else: + t0 = time.perf_counter() + ext_result = await extract(chunks) + stat.sec_extract = time.perf_counter() - t0 + stat.entities_raw = len(ext_result.entities) + stat.relations = len(ext_result.relations) + + # 3. Linking β€” bge-m3 NED + if ext_result.entities: + t0 = time.perf_counter() + link_result = link_entities(ext_result.entities) + stat.sec_linking = time.perf_counter() - t0 + stat.entities_grouped = link_result.grouped_count + else: + from kg.linking import LinkingResult + link_result = LinkingResult(groups=[], raw_count=0, grouped_count=0) + + # 4. Neo4j 적재 β€” Layer A + Layer B + MENTIONS + t0 = time.perf_counter() + build_layer_a(document, client=client, create_indexes=create_indexes) + build_layer_b(link_result, ext_result.relations, client=client, create_indexes=False) + mentions_result = link_chunks_to_entities(link_result, client=client) + stat.sec_neo4j = time.perf_counter() - t0 + stat.mentions = mentions_result.mentions_merged + + stat.success = True + + except Exception as exc: + stat.error = f"{type(exc).__name__}: {exc}" + logger.error("[%s] μ‹€νŒ¨: %s", path.name, stat.error) + logger.debug(traceback.format_exc()) + + stat.sec_total = time.perf_counter() - started_total + return stat + + +def _print_table(stats: list[DocStat]) -> None: + """κ²°κ³Ό ν‘œλ₯Ό stdout 에 좜λ ₯. weekly-log 에 κ·ΈλŒ€λ‘œ 볡사 κ°€λŠ₯ν•œ markdown.""" + print() + print("## 8λ¬Έμ„œ 일괄 적재 κ²°κ³Ό") + print() + print("| 파일 | 포맷 | νƒ€μž… | sec | chunks(T/B) | ent(rawβ†’grp) | rel | mentions | parse | extract | link | neo4j | total | ok |") + print("|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|") + for s in stats: + ok = "βœ…" if s.success else "❌" + rel_str = str(s.relations) if s.success else (s.error or "")[:30] + print( + f"| {s.filename} | {s.source_format} | {s.doc_type} | " + f"{s.sections} | {s.chunks_text}/{s.chunks_table} | " + f"{s.entities_raw}β†’{s.entities_grouped} | " + f"{s.relations} | {s.mentions} | " + f"{s.sec_parse:.1f} | {s.sec_extract:.1f} | " + f"{s.sec_linking:.1f} | {s.sec_neo4j:.1f} | " + f"{s.sec_total:.1f} | {ok} |" + ) + print() + + ok_count = sum(1 for s in stats if s.success) + print(f"총 {len(stats)}λ¬Έμ„œ Β· 성곡 {ok_count} Β· μ‹€νŒ¨ {len(stats) - ok_count}") + print(f"전체 μ†Œμš”: {sum(s.sec_total for s in stats):.1f}s") + print(f"총 청크 (text): {sum(s.chunks_text for s in stats)}") + print(f"총 entity raw: {sum(s.entities_raw for s in stats)}") + print(f"총 entity grouped: {sum(s.entities_grouped for s in stats)}") + print(f"총 relation: {sum(s.relations for s in stats)}") + print(f"총 MENTIONS: {sum(s.mentions for s in stats)}") + + +async def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--limit", type=int, default=None, + help="처음 N λ¬Έμ„œλ§Œ 처리 (sanity 용). κΈ°λ³Έ: μ „λΆ€.", + ) + parser.add_argument( + "--pattern", type=str, default="*", + help="파일λͺ… glob νŒ¨ν„΄ (예: '*.pdf'). κΈ°λ³Έ: 전체.", + ) + parser.add_argument( + "--docs-dir", type=Path, default=DOCS_DIR, + help="λ¬Έμ„œ 디렉토리. κΈ°λ³Έ: eval/dataset/documents/", + ) + args = parser.parse_args() + + if not args.docs_dir.exists(): + logger.error("λ¬Έμ„œ 디렉토리 μ—†μŒ: %s", args.docs_dir) + return 1 + + files = sorted(args.docs_dir.glob(args.pattern)) + files = [p for p in files if p.is_file()] + if args.limit: + files = files[: args.limit] + + if not files: + logger.error("μ²˜λ¦¬ν•  파일 μ—†μŒ (dir=%s pattern=%s)", args.docs_dir, args.pattern) + return 1 + + logger.info("=== 8λ¬Έμ„œ 일괄 적재 μ‹œμž‘ β€” %d 파일 ===", len(files)) + for p in files: + logger.info(" - %s", p.name) + + stats: list[DocStat] = [] + + with Neo4jClient() as client: + client.verify_connectivity() + + for idx, path in enumerate(files): + logger.info("\n=== [%d/%d] %s ===", idx + 1, len(files), path.name) + # 첫 λ¬Έμ„œλ§Œ 인덱슀 생성 μ‹œλ„ (멱등이라 맀번 ν˜ΈμΆœν•΄λ„ OK μ§€λ§Œ μ ˆμ•½) + stat = await _process_one( + path, client=client, create_indexes=(idx == 0), + ) + stats.append(stat) + + _print_table(stats) + return 0 if all(s.success for s in stats) else 1 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/kg/test_builder_layer_a.py b/tests/kg/test_builder_layer_a.py new file mode 100644 index 0000000..5074a78 --- /dev/null +++ b/tests/kg/test_builder_layer_a.py @@ -0,0 +1,389 @@ +"""kg/builder.py 의 Layer A ν•¨μˆ˜ + MENTIONS λ§€ν•‘ λ‹¨μœ„ ν…ŒμŠ€νŠΈ. + +검증 포인트 (#17): +- build_layer_a: Document/Section/Chunk/Table MERGE 호좜 + 관계 4μ’… +- 청크 NEXT 관계: Section λ‚΄ μˆœμ„œλŒ€λ‘œλ§Œ (Section 경계 λ„˜μ§€ μ•ŠμŒ) +- link_chunks_to_entities: parse_global_id 둜 chunk_id μΆ”μΆœ ν›„ MENTIONS +- chunk_id μ—†λŠ” ꡬ식 local_id λŠ” graceful drop (5/10 ν˜Έν™˜) + +ν…ŒμŠ€νŠΈ 방식: mock Neo4jClient β€” μ‹€μ œ DB 없이 client.write() 호좜 인자 검증. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import pytest + +from ingestion.models import ( + Chunk as LayerAChunk, + ChunkType, + DocType, + Document, + Section, + SectionLabel, + SourceFormat, + Table, +) +from kg.builder import build_layer_a, link_chunks_to_entities +from kg.linking import EntityGroup, LinkingResult +from kg.ontology import EntityType, ExtractedEntity + + +# ── Mock Neo4jClient ────────────────────────────────────── +@dataclass +class _MockClient: + """Neo4jClient μΈν„°νŽ˜μ΄μŠ€ 흉내 β€” write() ν˜ΈμΆœμ„ 기둝만 함.""" + + writes: list[tuple[str, dict]] = field(default_factory=list) + + def write(self, query: str, **params) -> list: + self.writes.append((query, params)) + return [] + + def read(self, query: str, **params) -> list: + return [] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _writes_with_keyword(client: _MockClient, keyword: str) -> list[tuple[str, dict]]: + """write 호좜 쀑 query 에 keyword ν¬ν•¨λœ κ²ƒλ§Œ.""" + return [(q, p) for q, p in client.writes if keyword in q] + + +# ── ν”½μŠ€μ²˜: 두산λ°₯μΊ£ mini Document ────────────────────────── +@pytest.fixture +def doosan_doc() -> Document: + """1 Document + 2 Section + 3 Chunk + 1 Table 의 λ―Έλ‹ˆ λ¬Έμ„œ.""" + section1 = Section( + id="doc1:s000", + original_section="3λΆ„κΈ° 싀적", + doc_type=DocType.REPORT, + label=SectionLabel.PERFORMANCE, + heading_level=2, + order_index=0, + page_start=3, + page_end=4, + chunks=[ + LayerAChunk( + id="doc1:c0000", + text="두산λ°₯μΊ£ μ˜μ—…μ΄μ΅ 35% κ°μ†Œ.", + chunk_type=ChunkType.TEXT, + char_count=20, + page=3, + order_index=0, + ), + LayerAChunk( + id="doc1:c0001", + text="뢁미 고금리 μž₯κΈ°ν™”κ°€ 원인.", + chunk_type=ChunkType.TEXT, + char_count=18, + page=3, + order_index=1, + ), + ], + tables=[ + Table( + id="doc1:t000", + raw_markdown="| ν•­λͺ© | κ°’ |\n|---|---|\n| μ˜μ—…μ΄μ΅ | 1.2μ‘° |", + row_count=3, + column_count=2, + page=4, + order_index=0, + ), + ], + ) + section2 = Section( + id="doc1:s001", + original_section="전망", + doc_type=DocType.REPORT, + label=SectionLabel.OUTLOOK, + heading_level=2, + order_index=1, + chunks=[ + LayerAChunk( + id="doc1:c0002", + text="2026 회볡 전망.", + chunk_type=ChunkType.TEXT, + char_count=10, + page=5, + order_index=2, + ), + ], + ) + return Document( + id="doc1", + filename="ν•œν™”_두산λ°₯μΊ£.pdf", + source_format=SourceFormat.PDF, + doc_type=DocType.REPORT, + total_pages=5, + publisher="ν•œν™”νˆ¬μžμ¦κΆŒ", + subject="두산λ°₯μΊ£", + fiscal_year=2024, + sections=[section1, section2], + ) + + +# ── build_layer_a ν…ŒμŠ€νŠΈ ────────────────────────────────── +def test_build_layer_a_node_counts(doosan_doc: Document): + """Document 1 + Section 2 + Chunk 3 + Table 1 λ…Έλ“œ MERGE λ°œμƒ.""" + client = _MockClient() + result = build_layer_a(doosan_doc, client=client, create_indexes=False) + + assert result.documents_merged == 1 + assert result.sections_merged == 2 + assert result.chunks_merged == 3 + assert result.tables_merged == 1 + + +def test_build_layer_a_document_params(doosan_doc: Document): + """Document MERGE 에 publisher / fiscal_year λ“± 메타 μ •ν™•νžˆ 전달.""" + client = _MockClient() + build_layer_a(doosan_doc, client=client, create_indexes=False) + + doc_writes = _writes_with_keyword(client, "MERGE (d:Document") + assert len(doc_writes) == 1 + _, params = doc_writes[0] + assert params["id"] == "doc1" + assert params["filename"] == "ν•œν™”_두산λ°₯μΊ£.pdf" + assert params["doc_type"] == "report" + assert params["source_format"] == "pdf" + assert params["publisher"] == "ν•œν™”νˆ¬μžμ¦κΆŒ" + assert params["fiscal_year"] == 2024 + + +def test_build_layer_a_section_label_serialized(doosan_doc: Document): + """Section.label (enum) β†’ λ¬Έμžμ—΄ κ°’μœΌλ‘œ 직렬화.""" + client = _MockClient() + build_layer_a(doosan_doc, client=client, create_indexes=False) + + section_writes = _writes_with_keyword(client, "MERGE (s:Section") + assert len(section_writes) == 2 + labels = {p["label"] for _, p in section_writes} + assert labels == {"Performance", "Outlook"} + + +def test_build_layer_a_section_label_none_safe(): + """Section.label 이 None 이어도 적재 κ°€λŠ₯ (W3 λ―ΈλΆ„λ₯˜ 청크 λŒ€μ‘).""" + doc = Document( + id="d", filename="t.pdf", source_format=SourceFormat.PDF, + doc_type=DocType.IR, total_pages=1, + sections=[ + Section( + id="s", original_section="raw", doc_type=DocType.IR, + label=None, # ← λ―ΈλΆ„λ₯˜ + ), + ], + ) + client = _MockClient() + build_layer_a(doc, client=client, create_indexes=False) + section_writes = _writes_with_keyword(client, "MERGE (s:Section") + _, params = section_writes[0] + assert params["label"] is None + + +def test_build_layer_a_next_relations_only_within_section(doosan_doc: Document): + """NEXT κ΄€κ³„λŠ” Section λ‚΄μ—μ„œλ§Œ. Section 경계 λ„˜μ§€ μ•ŠμŒ.""" + client = _MockClient() + result = build_layer_a(doosan_doc, client=client, create_indexes=False) + + # section1: chunks c0000 β†’ c0001 (1개), section2: c0002 단독 (0개) + assert result.next_relations == 1 + + next_writes = _writes_with_keyword(client, "[:NEXT]") + assert len(next_writes) == 1 + _, params = next_writes[0] + assert params["prev_id"] == "doc1:c0000" + assert params["next_id"] == "doc1:c0001" + + +def test_build_layer_a_table_attached_to_section(doosan_doc: Document): + """CONTAINS_TABLE 관계가 μ˜¬λ°”λ₯Έ Section 에 μ—°κ²°.""" + client = _MockClient() + build_layer_a(doosan_doc, client=client, create_indexes=False) + + table_writes = _writes_with_keyword(client, "[:CONTAINS_TABLE]") + assert len(table_writes) == 1 + _, params = table_writes[0] + assert params["section_id"] == "doc1:s000" + assert params["table_id"] == "doc1:t000" + + +def test_build_layer_a_idempotent_uses_merge(doosan_doc: Document): + """λͺ¨λ“  쿼리가 MERGE 기반 (idempotent). CREATE 단독 μ—†μŒ.""" + client = _MockClient() + build_layer_a(doosan_doc, client=client, create_indexes=False) + + for query, _ in client.writes: + # CREATE CONSTRAINT/INDEX λŠ” create_indexes=False 라 호좜 μ•ˆ 됨 + assert "MERGE" in query + + +# ── link_chunks_to_entities ν…ŒμŠ€νŠΈ ──────────────────────── +def _make_group(group_id: str, type_: EntityType, members: list[ExtractedEntity]) -> EntityGroup: + return EntityGroup( + group_id=group_id, + type=type_, + representative_name=members[0].canonical if members else "", + members=members, + ) + + +def test_link_chunks_to_entities_basic(): + """각 EntityGroup.member λ§ˆλ‹€ chunk_id λ₯Ό μΆ”μΆœν•΄ MENTIONS MERGE.""" + linking = LinkingResult( + groups=[ + _make_group( + "grp_001", + EntityType.COMPANY, + [ + ExtractedEntity( + local_id="doc1:c0000__ent_001", + type=EntityType.COMPANY, + canonical="두산λ°₯μΊ£", + source_span="두산λ°₯μΊ£", + section="3λΆ„κΈ° 싀적", + ), + ], + ), + _make_group( + "grp_002", + EntityType.METRIC, + [ + ExtractedEntity( + local_id="doc1:c0000__ent_002", + type=EntityType.METRIC, + canonical="μ˜μ—…μ΄μ΅ -35%", + source_span="μ˜μ—…μ΄μ΅μ΄ μ „λ…„ 동기 λŒ€λΉ„ 35% κ°μ†Œ", + section="3λΆ„κΈ° 싀적", + ), + ], + ), + ], + raw_count=2, + grouped_count=2, + ) + + client = _MockClient() + result = link_chunks_to_entities(linking, client=client) + + assert result.mentions_merged == 2 + assert result.mentions_dropped == 0 + + # 각 MENTIONS write κ°€ μ˜¬λ°”λ₯Έ chunk_id + group_id λ₯Ό κ°€μ§€λŠ”μ§€ + mentions = _writes_with_keyword(client, "[:MENTIONS]") + assert len(mentions) == 2 + pairs = {(p["chunk_id"], p["group_id"]) for _, p in mentions} + assert pairs == { + ("doc1:c0000", "grp_001"), + ("doc1:c0000", "grp_002"), + } + + +def test_link_chunks_to_entities_dedupe_same_chunk(): + """ν•œ κ·Έλ£Ή λ‚΄ 같은 청크의 member κ°€ μ—¬λŸ¬ κ°œμ—¬λ„ 청크당 1회만 MENTIONS.""" + linking = LinkingResult( + groups=[ + _make_group( + "grp_001", + EntityType.COMPANY, + [ + # 같은 μ²­ν¬μ—μ„œ 같은 그룹의 alias κ°€ 2번 λ“±μž₯ (λ“œλ¬Όμ§€λ§Œ κ°€λŠ₯) + ExtractedEntity( + local_id="doc1:c0000__ent_001", + type=EntityType.COMPANY, canonical="두산λ°₯μΊ£", + source_span="두산λ°₯μΊ£", section="3λΆ„κΈ°", + ), + ExtractedEntity( + local_id="doc1:c0000__ent_003", + type=EntityType.COMPANY, canonical="Doosan Bobcat", + source_span="Doosan Bobcat", section="3λΆ„κΈ°", + ), + ], + ), + ], + raw_count=2, + grouped_count=1, + ) + + client = _MockClient() + result = link_chunks_to_entities(linking, client=client) + + # 청크 1개, κ·Έλ£Ή 1개 β†’ MENTIONS 1번만 + assert result.mentions_merged == 1 + mentions = _writes_with_keyword(client, "[:MENTIONS]") + assert len(mentions) == 1 + + +def test_link_chunks_to_entities_multi_chunk_same_group(): + """λ‹€λ₯Έ 청크의 member 듀은 각각 MENTIONS β€” 청크당 1λ²ˆμ”©.""" + linking = LinkingResult( + groups=[ + _make_group( + "grp_001", + EntityType.COMPANY, + [ + ExtractedEntity( + local_id="doc1:c0000__ent_001", + type=EntityType.COMPANY, canonical="λ―Έλž˜μ—μ…‹", + source_span="λ―Έλž˜μ—μ…‹", section="1Q", + ), + ExtractedEntity( + local_id="doc1:c0010__ent_001", + type=EntityType.COMPANY, canonical="λ―Έλž˜μ—μ…‹μ¦κΆŒ", + source_span="λ―Έλž˜μ—μ…‹μ¦κΆŒ", section="2Q", + ), + ExtractedEntity( + local_id="doc1:c0020__ent_002", + type=EntityType.COMPANY, canonical="λ―Έλž˜μ—μ…‹", + source_span="λ―Έλž˜μ—μ…‹", section="3Q", + ), + ], + ), + ], + raw_count=3, + grouped_count=1, + ) + + client = _MockClient() + result = link_chunks_to_entities(linking, client=client) + + # 3개 청크 β†’ 3개 MENTIONS + assert result.mentions_merged == 3 + mentions = _writes_with_keyword(client, "[:MENTIONS]") + chunk_ids = sorted({p["chunk_id"] for _, p in mentions}) + assert chunk_ids == ["doc1:c0000", "doc1:c0010", "doc1:c0020"] + + +def test_link_chunks_to_entities_legacy_local_id_dropped(): + """chunk_id prefix μ—†λŠ” ꡬ식 local_id λŠ” graceful drop (5/10 sanity ν˜Έν™˜).""" + linking = LinkingResult( + groups=[ + _make_group( + "grp_001", + EntityType.COMPANY, + [ + ExtractedEntity( + local_id="ent_001", # ← prefix μ—†μŒ + type=EntityType.COMPANY, canonical="두산λ°₯μΊ£", + source_span="두산λ°₯μΊ£", section="3λΆ„κΈ°", + ), + ], + ), + ], + raw_count=1, + grouped_count=1, + ) + + client = _MockClient() + result = link_chunks_to_entities(linking, client=client) + + assert result.mentions_merged == 0 + assert result.mentions_dropped == 1 + # μ‹€μ œ write 호좜 μ—†μŒ + assert _writes_with_keyword(client, "[:MENTIONS]") == []