feat: refresh material names in background every two days

capabilities/am516-delivery-prediction/scripts/refresh_material_names.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import argparse
+import fcntl
 import hashlib
 import importlib.util
 import json
@@ -21,10 +22,13 @@ sys.path.insert(0, str(PROJECT_ROOT))
 from shared.runtime.material_name_cache import (
     MATERIAL_NAME_CACHE_FILE,
     MaterialNameCacheError,
+    load_material_names,
     material_names_from_records,
     material_names_hash,
 )
 
+BUNDLED_CLIENT = PROJECT_ROOT / "shared/runtime/kingdee_materials.py"
+DEFAULT_STATE = PROJECT_ROOT / "logs/material_names_refresh_state.json"
 
 def read_reference(path: Path) -> tuple[list[dict[str, Any]], str]:
     payload = json.loads(path.read_text(encoding="utf-8"))
@@ -83,20 +87,7 @@ def query_materials(client_path: Path) -> tuple[list[dict[str, Any]], str]:
     return rows, snapshot_at
 
 
-def write_cache(records: list[dict[str, Any]], snapshot_at: str, output: Path) -> dict[str, Any]:
-    materials = material_names_from_records(records)
-    payload = {
-        "schema_version": 1,
-        "source": {
-            "system": "Kingdee K3 Cloud", "form_id": "BD_MATERIAL",
-            "field_keys": ["FNumber", "FName"], "filter": "",
-            "snapshot_at": snapshot_at, "record_count": len(records),
-        },
-        "generated_at": datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds"),
-        "material_count": len(materials),
-        "content_sha256": material_names_hash(materials),
-        "materials": materials,
-    }
+def write_json_atomic(output: Path, payload: dict[str, Any]) -> None:
     output.parent.mkdir(parents=True, exist_ok=True)
     temp_path = None
     try:
@@ -110,33 +101,108 @@ def write_cache(records: list[dict[str, Any]], snapshot_at: str, output: Path) -
     finally:
         if temp_path is not None:
             temp_path.unlink(missing_ok=True)
+
+
+def write_cache(
+    records: list[dict[str, Any]], snapshot_at: str, output: Path, *, merge_existing: bool = False,
+) -> dict[str, Any]:
+    materials = material_names_from_records(records)
+    retained_count = 0
+    if merge_existing and output.exists():
+        previous = load_material_names(output)
+        retained_count = len(previous.keys() - materials.keys())
+        materials = dict(sorted({**previous, **materials}.items()))
+        if materials == previous:
+            return json.loads(output.read_text(encoding="utf-8"))
+    payload = {
+        "schema_version": 1,
+        "source": {
+            "system": "Kingdee K3 Cloud", "form_id": "BD_MATERIAL",
+            "field_keys": ["FNumber", "FName"], "filter": "",
+            "snapshot_at": snapshot_at, "record_count": len(records),
+        },
+        "generated_at": datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds"),
+        "material_count": len(materials),
+        "content_sha256": material_names_hash(materials),
+        "materials": materials,
+    }
+    if retained_count:
+        payload["source"]["retained_missing_count"] = retained_count
+    write_json_atomic(output, payload)
     return payload
 
 
+def refresh_once(
+    *, output: Path, state_file: Path, due_only: bool, interval_hours: int,
+    client_path: Path, reference_json: Path | None = None, current_time: datetime | None = None,
+) -> dict[str, Any]:
+    now = current_time or datetime.now(timezone(timedelta(hours=8)))
+    if due_only and state_file.exists():
+        try:
+            state = json.loads(state_file.read_text(encoding="utf-8"))
+            last = datetime.fromisoformat(state["last_success_at"])
+            if last.tzinfo is None:
+                raise ValueError
+            next_due = last + timedelta(hours=interval_hours)
+        except (OSError, ValueError, KeyError, TypeError):
+            next_due = now  # Rebuild invalid local state only after a successful refresh.
+        if now < next_due:
+            return {"ok": True, "status": "NOT_DUE", "next_due_at": next_due.isoformat()}
+    previous = load_material_names(output) if output.exists() else {}
+    records, snapshot_at = read_reference(reference_json) if reference_json else query_materials(client_path)
+    incoming = material_names_from_records(records)
+    added_count = len(incoming.keys() - previous.keys())
+    renamed_count = sum(previous[code] != name for code, name in incoming.items() if code in previous)
+    payload = write_cache(records, snapshot_at, output, merge_existing=due_only)
+    completed_at = current_time or datetime.now(timezone(timedelta(hours=8)))
+    result = {
+        "ok": True, "status": "UPDATED" if added_count or renamed_count else "UNCHANGED",
+        "source_record_count": len(records), "material_count": payload["material_count"],
+        "added_count": added_count, "renamed_count": renamed_count,
+        "retained_missing_count": len(previous.keys() - incoming.keys()) if due_only else 0,
+        "snapshot_at": snapshot_at, "output": str(output),
+    }
+    if due_only:
+        write_json_atomic(state_file, {
+            "last_success_at": completed_at.isoformat(timespec="seconds"),
+            "last_result": result,
+        })
+    return result
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
-    source = parser.add_mutually_exclusive_group(required=True)
+    source = parser.add_mutually_exclusive_group()
     source.add_argument("--reference-json", type=Path, help="Existing verified full material-master cache")
     source.add_argument("--kingdee-client", type=Path, help="Approved kingdee_query.py for a live read-only refresh")
     parser.add_argument("--output", type=Path, default=MATERIAL_NAME_CACHE_FILE)
+    parser.add_argument("--due", action="store_true", help="Query only after the successful refresh interval")
+    parser.add_argument("--interval-hours", type=int, default=48)
+    parser.add_argument("--state-file", type=Path, default=DEFAULT_STATE)
     args = parser.parse_args()
+    if args.interval_hours <= 0 or (args.due and args.reference_json):
+        parser.error("--interval-hours 必须为正数；--due 只允许实时 ERP 刷新。")
+    args.state_file.parent.mkdir(parents=True, exist_ok=True)
+    before = hashlib.sha256(args.output.read_bytes()).hexdigest() if args.output.exists() else None
     try:
-        records, snapshot_at = (
-            read_reference(args.reference_json)
-            if args.reference_json is not None
-            else query_materials(args.kingdee_client)
-        )
-        payload = write_cache(records, snapshot_at, args.output)
+        with args.state_file.with_suffix(".lock").open("a") as lock:
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                print(json.dumps({"ok": True, "status": "ALREADY_RUNNING"}))
+                return 0
+            result = refresh_once(
+                output=args.output, state_file=args.state_file, due_only=args.due,
+                interval_hours=args.interval_hours, client_path=args.kingdee_client or BUNDLED_CLIENT,
+                reference_json=args.reference_json,
+            )
     except Exception as exc:
         # Kingdee exceptions may contain server text: do not print arbitrary content.
-        reason = str(exc) if isinstance(exc, MaterialNameCacheError) else f"读取物料数据失败（{type(exc).__name__}），请检查只读连接和凭证配置。"
-        print(json.dumps({"ok": False, "error": reason, "previous_cache_preserved": True}, ensure_ascii=False))
+        reason = str(exc) if isinstance(exc, MaterialNameCacheError) or type(exc).__name__ == "KingdeeMaterialError" else f"读取物料数据失败（{type(exc).__name__}），请检查只读连接和凭证配置。"
+        after = hashlib.sha256(args.output.read_bytes()).hexdigest() if args.output.exists() else None
+        print(json.dumps({"ok": False, "error": reason, "previous_cache_preserved": before == after}, ensure_ascii=False))
         return 1
-    print(json.dumps({
-        "ok": True, "source_record_count": payload["source"]["record_count"],
-        "material_count": payload["material_count"], "snapshot_at": snapshot_at,
-        "output": str(args.output),
-    }, ensure_ascii=False))
+    print(json.dumps(result, ensure_ascii=False))
     return 0