BJ_ZTEITS-Developers / parking-cloud

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
P2 审计脚本：把存量代码真正需要的 `${占位符}` 与「新工程提供的配置」对账。

## 为什么需要它

老工程的 `application*.properties` 是**按模块各一份**的（`zteits-park` / `zteits-order` /
`zteits-pay` / `zteits-charge` … 共十几份），而搬迁红线第 9 条「禁止修改
application-*.properties 中的环境配置」+ `tools/p2_import_legacy.py` 的策略，
决定了这些文件**故意不搬** —— 环境配置以新工程 `parking-server/src/main/resources/*.yaml` 为准。

于是必然出现缺口：存量代码里 `@Value("${xxx}")` 的**业务参数**（不是连接串）在新工程里没人提供，
启动报

    java.lang.IllegalArgumentException: Could not resolve placeholder 'xxx' in value "${xxx}"

而且**一次只报第一个** —— 靠启动试错要启 N 次。本脚本一次列全，并给出老工程里的原值。

## 三类假阳性（都是本脚本第一版踩过的坑，缺一不可）

| 假阳性 | 例子 | 为什么不是缺口 |
|---|---|---|
| 注解被注释掉了 | `AliOssClientConfig` 里 7 个 `ali.oss.*` 被 `/* */` 整段注释（实际改从数据库 `SysCodeValueCacheUtil` 读） | 代码里根本没有这个注解 |
| `@Value` 标在 **static** 字段上 | `Wechatpayapiv3Config` 的 4 个 `wechatpay.apiv3.*` | Spring **不会**注入 static 字段，值来自字段初始化表达式 |
| 类根本不是 Bean | 包不在启动类扫描范围内、或类上没有组件注解、或已被 `excludeFilters` 排除 | 容器里没有这个类，`@Value` 永远不会被解析 |

所以判定链是：**注解真的存在（去注释）→ 不是 static → 类会被注册成 Bean → 才是「必需」**。

## 扫描范围是怎么算出来的

启动类 `ParkingServerApplication` 的 `@ComponentScan.basePackages`（含 `${parking.info.base-package}`
这类占位符，用 `application.yaml` 里的值代换）+ **源码里嵌套的 `@ComponentScan`**（递归到不动点，
无参数的那种按「本类所在包」算）。这套计算同时也是对手写扫描列表的一次校验：
`--show-scan` 会把它解析出的包逐个打出来。

## 用法

    python3 tools/p2_audit_placeholders.py            # 只列「必需但缺失」与「有默认值需核对」
    python3 tools/p2_audit_placeholders.py --all      # 连「已提供」「判为无需配置」一起列出
    python3 tools/p2_audit_placeholders.py --emit /tmp/legacy-params.properties
                                                      # 生成可直接搬进 parking-server 的配置草稿
"""

import argparse
import csv
import glob
import os
import re
import sys
from collections import defaultdict

CLOUD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LEGACY = "/Users/wangbiao/IdeaProjects/parking/zteits-clouds"

SCAN_DIRS = [
    "parking-server", "parking-framework", "parking-module-api", "parking-module-infra",
    "parking-module-system", "parking-module-park", "parking-module-order",
    "parking-module-pay", "parking-module-bill", "parking-module-charge",
    "parking-module-business",
]

APP_JAVA = "parking-server/src/main/java/com/zteits/parking/server/ParkingServerApplication.java"

VALUE_RE = re.compile(r'@Value\s*\(\s*(?:value\s*=\s*)?"\$\{([^}]*)\}"')
NESTED_RE = re.compile(r'\$\{([^}]*)\}')
PKG_RE = re.compile(r'^\s*package\s+([\w.]+)\s*;', re.M)
YAML_KV_RE = re.compile(r'^(\s*)([A-Za-z0-9_.\-]+):(\s|$)')
PROP_KV_RE = re.compile(r'^\s*([A-Za-z0-9_.\-]+)\s*[=:]\s*(.*)$')

STEREOTYPES = ("@Component", "@Configuration", "@Service", "@Repository", "@Controller",
               "@RestController", "@ConfigurationProperties")
SCAN_ANNO_RE = re.compile(
    r'@ComponentScans?\s*\('  # 只处理带参数的；无参数的单独处理
)

# 值像「凭据」的键 → 不把老工程的值写进仓库，改成环境变量占位
SECRET_HINTS = ("password", "passwd", "secret", "accesskey", "privatekey", "apiv3key",
                "token", "cert.pass", "appid", "private-key", "sdkkey", "mchkey")


# --------------------------------------------------------------------------- #
# 基础工具
# --------------------------------------------------------------------------- #
def strip_comments(src):
    """把注释内容换成空格（保留换行与偏移，行号仍然可用），字符串字面量原样保留。

    只处理 // 与 /* */；不处理字符串里的转义细节 —— 本脚本只关心 @Value("${...}")，
    这种精度足够。"""
    out = []
    i, n = 0, len(src)
    while i < n:
        c = src[i]
        if c == '"' or c == "'":  # 字符串/字符字面量，整段原样保留
            q = c
            out.append(c)
            i += 1
            while i < n:
                if src[i] == "\\":
                    out.append(src[i:i + 2])
                    i += 2
                    continue
                out.append(src[i])
                if src[i] == q:
                    i += 1
                    break
                i += 1
            continue
        if c == "/" and i + 1 < n and src[i + 1] == "/":
            while i < n and src[i] != "\n":
                out.append(" ")
                i += 1
            continue
        if c == "/" and i + 1 < n and src[i + 1] == "*":
            out.append("  ")
            i += 2
            while i + 1 < n and not (src[i] == "*" and src[i + 1] == "/"):
                out.append("\n" if src[i] == "\n" else " ")
                i += 1
            out.append("  ")
            i += 2
            continue
        out.append(c)
        i += 1
    return "".join(out)


def balanced(text, open_idx):
    """open_idx 指向 '('，返回配对的 ')' 之后的位置，以及括号内部文本。"""
    depth = 0
    for i in range(open_idx, len(text)):
        if text[i] == "(":
            depth += 1
        elif text[i] == ")":
            depth -= 1
            if depth == 0:
                return i + 1, text[open_idx + 1:i]
    return len(text), text[open_idx + 1:]


def brace_span(text, open_idx):
    """open_idx 指向 '{'，配对着色后返回 (内部文本, 结束位置)。

    ⚠️ 不能用 `\\{(.*?)\\}` 这种正则：值里常有 `${...}` 占位符，
    **非贪婪匹配会在第一个 `}`（也就是 `${...}` 的闭合括号）就截断**，
    这个坑本脚本踩过 —— 表现为「明明有 basePackages 却一个包都没解析出来」。"""
    depth = 0
    for i in range(open_idx, len(text)):
        if text[i] == "{":
            depth += 1
        elif text[i] == "}":
            depth -= 1
            if depth == 0:
                return text[open_idx + 1:i], i + 1
    return text[open_idx + 1:], len(text)


def split_placeholder(expr):
    """`a:b` → ('a','b')；`a` → ('a', None)。与 Spring 的 PlaceholderHelper 行为一致。"""
    if ":" in expr:
        k, v = expr.split(":", 1)
        return k.strip(), v
    return expr.strip(), None


# --------------------------------------------------------------------------- #
# 1) 新工程的配置提供方（yaml / properties）
# --------------------------------------------------------------------------- #
def flatten_yaml(path):
    """极简 YAML 展平：只处理「缩进 + key: value」和 `key:` 开块。手写规整缩进够用。"""
    out, stack = {}, []
    with open(path, encoding="utf-8", errors="replace") as fh:
        for line in fh:
            if not line.strip() or line.lstrip().startswith("#"):
                continue
            m = YAML_KV_RE.match(line)
            if not m:
                continue
            indent, key = len(m.group(1)), m.group(2)
            while stack and stack[-1][0] >= indent:
                stack.pop()
            full = ".".join([k for _, k in stack] + [key])
            rest = line.split(":", 1)[1].split(" #", 1)[0].strip()
            if rest and rest not in ("|", ">", "|-", ">-"):
                out[full] = rest
            stack.append((indent, key))
    return out


def scan_provided():
    """新工程「已提供」的键 = 它自己的 yaml + 这些 yaml 用 spring.config.import 引进来的文件。

    ⚠️ 第二个来源是后补的，缺了它就是本脚本踩过的坑：
    脚本原本只 glob `application*.yaml`，而它自己的 `--emit` 生成的
    `legacy-params.properties` 是**通过 `spring.config.import` 生效**的 ——
    「自己开出的方子自己不认」，于是那 36 个键全被判成「必需但未提供」，
    退出码恒为 1（假阳性），而应用其实启动得好好的。
    所以这里按 `spring.config.import` 实际声明的文件名去读，而不是把文件名写死：
    import 列表改了，这里跟着改。
    """
    provided = defaultdict(list)
    res_dir = os.path.join(CLOUD, "parking-server/src/main/resources")
    yamls = []
    for pat in ("application.yaml", "application.yml", "application-*.yaml", "application-*.yml"):
        yamls += sorted(glob.glob(os.path.join(res_dir, pat)))

    flats = {}
    for p in yamls:
        flats[p] = flatten_yaml(p)
        for k, v in flats[p].items():
            provided[k].append((os.path.basename(p), v))

    for p in yamls:
        for raw in flats[p].get("spring.config.import", "").split(","):
            name = strip_location(raw)
            full = os.path.join(res_dir, name) if name else ""
            if not name or not os.path.isfile(full):
                continue
            keys = (flatten_yaml(full) if name.endswith((".yaml", ".yml"))
                    else read_properties(full))
            for k, v in keys.items():
                provided[k].append((name, v))
    return provided


def strip_location(loc):
    """`classpath:legacy-params.properties` / `optional:classpath:x.yml` → 去前缀去引号。"""
    loc = loc.strip().strip("'\"")
    while True:
        for pfx in ("optional:", "classpath*:", "classpath:", "file:"):
            if loc.startswith(pfx):
                loc = loc[len(pfx):]
                break
        else:
            return loc


def read_properties(p):
    out = {}
    with open(p, encoding="utf-8", errors="replace") as fh:
        for line in fh:
            s = line.strip()
            if not s or s[0] in "#!":
                continue
            m = PROP_KV_RE.match(line)
            if m:
                out[m.group(1)] = m.group(2).split(" #", 1)[0].strip()
    return out


def scan_legacy():
    """→ {key: {(模块, 文件): 值}}（含所有 profile 文件，用于发现「同键不同值」）"""
    legacy = defaultdict(dict)
    for p in glob.glob(os.path.join(LEGACY, "*", "src/main/resources/application*.properties")):
        rel = os.path.relpath(p, LEGACY)
        module, fname = rel.split(os.sep)[0], os.path.basename(p)
        with open(p, encoding="utf-8", errors="replace") as fh:
            for line in fh:
                s = line.strip()
                if not s or s[0] in "#!":
                    continue
                m = PROP_KV_RE.match(line)
                if m:
                    legacy[m.group(1)][(module, fname)] = m.group(2).strip()
    return legacy


def legacy_baseline(legacy, key):
    """老工程的基线值：优先 `application.properties`（那是默认环境），否则取任一。"""
    d = legacy.get(key)
    if not d:
        return None, []
    for (module, fname), v in d.items():
        if fname == "application.properties":
            return (module, v), sorted({x for x in d.values()})
    # 没有默认环境文件（如 zteits-charge 只有 application-danyang.properties）→ 取第一个
    ((module, fname), v) = sorted(d.items())[0]
    return (module, v), sorted({x for x in d.values()})


# --------------------------------------------------------------------------- #
# 2) 扫描范围（哪些包里的类会成为 Bean）
# --------------------------------------------------------------------------- #
def resolve_placeholder_value(expr, provided):
    """把 `${parking.info.base-package}.server` 用新工程 yaml 里的值代换成 `com.zteits.parking.server`。
    解不出来就原样返回（带 `${}` 的字符串匹配不上任何包名，等于不算在扫描范围内 —— 宁可少算，
    由 --show-scan 打印出来人工核对）。"""
    m = re.fullmatch(r'\$\{([^}]*)\}(.*)', expr)
    if not m:
        return expr
    key, default = split_placeholder(m.group(1))
    suffix = m.group(2)
    for _, v in provided.get(key, []):
        if not v.startswith("${"):
            return v.strip("'\"") + suffix
    if default is not None:
        return default + suffix
    return expr


def java_sources():
    for d in SCAN_DIRS:
        root = os.path.join(CLOUD, d)
        for dirpath, _, files in os.walk(root):
            if os.sep + "target" + os.sep in dirpath + os.sep:
                continue
            for f in files:
                if f.endswith(".java"):
                    yield os.path.join(dirpath, f)


def scan_config():
    """→ (扫描包集合, 被 excludeFilters 排除的 FQCN 集合, 每个文件的包与是否为组件)"""
    provided = scan_provided()
    excluded = set()
    packs = set()

    def read_scan(text):
        """从一段源码里抽出 @ComponentScan 的 basePackages（无参数则返回 None=本类包）"""
        found = []
        for m in SCAN_ANNO_RE.finditer(text):
            end, inner = balanced(text, m.end() - 1)
            bp = re.search(r'basePackages\s*=\s*\{', inner)
            if bp:
                content, _ = brace_span(inner, bp.end() - 1)
                for raw in re.findall(r'"([^"]*)"', content):
                    found.append(resolve_placeholder_value(raw, provided))
            else:
                # 无参数（扫本类所在包）或 basePackageClasses 写法 —— 都交给调用方按「本类包」处理，
                # 后者会在 --show-scan 的结果里显得多算了几个包，人工核对即可
                found.append(None)
            if "excludeFilters" in inner:
                for cl in re.finditer(r'classes\s*=\s*\{', inner):
                    content, _ = brace_span(inner, cl.end() - 1)
                    for fq in re.findall(r'([A-Z][\w.]*\.[A-Z][\w.]*)\.class', content):
                        excluded.add(fq)
        return found

    app_path = os.path.join(CLOUD, APP_JAVA)
    app_text = strip_comments(open(app_path, encoding="utf-8").read())
    app_pkg = PKG_RE.search(app_text).group(1)
    for p in read_scan(app_text):
        packs.add(p or app_pkg)

    # 递归到不动点：扫描范围内的类若自己带 @ComponentScan（老代码里真有，见 AGENTS.md 坑 14）
    changed = True
    while changed:
        changed = False
        for path in java_sources():
            text = strip_comments(open(path, encoding="utf-8", errors="replace").read())
            pm = PKG_RE.search(text)
            if not pm:
                continue
            pkg = pm.group(1)
            if not any(pkg == p or pkg.startswith(p + ".") for p in packs):
                continue
            for target in read_scan(text):
                t = target or pkg
                if t not in packs:
                    packs.add(t)
                    changed = True
    return packs, excluded


def class_is_bean(text, pkg, excluded, packs):
    """类会成为 Bean 吗？（粗判：包在扫描范围内 + 有组件注解 + 未被排除）"""
    if not any(pkg == p or pkg.startswith(p + ".") for p in packs):
        return False, "包不在扫描范围内"
    cls = re.search(r'\b(?:class|interface)\s+(\w+)', text)
    fqn = f"{pkg}.{cls.group(1)}" if cls else None
    if fqn in excluded:
        return False, f"已被 excludeFilters 排除（{fqn}）"
    head = text[:cls.start()] if cls else text
    if not any(s in head for s in STEREOTYPES):
        return False, "类上没有组件注解"
    return True, ""


# --------------------------------------------------------------------------- #
# 3) 收集 @Value 需求
# --------------------------------------------------------------------------- #
def scan_required(packs, excluded):
    """→ required{key: [entry]}, skipped[entry], with_default[entry]"""
    required, skipped, with_default = defaultdict(list), [], []
    for path in java_sources():
        raw = open(path, encoding="utf-8", errors="replace").read()
        text = strip_comments(raw)
        pm = PKG_RE.search(text)
        if not pm:
            continue
        pkg = pm.group(1)
        is_bean, why = class_is_bean(text, pkg, excluded, packs)
        rel = os.path.relpath(path, CLOUD)
        for m in VALUE_RE.finditer(text):
            key, default = split_placeholder(m.group(1))
            if not key:
                continue
            lineno = text[:m.start()].count("\n") + 1
            # 注解之后到第一个 ';' 的片段里出现 static → Spring 不注入 static 字段
            tail = text[m.end():m.end() + 400]
            decl = tail.split(";", 1)[0]
            on_static = bool(re.search(r'\bstatic\b', decl))
            entry = dict(key=key, default=default, where=f"{rel}:{lineno}", why="")
            if on_static:
                entry["why"] = "注解标在 static 字段上，Spring 不注入"
                skipped.append(entry)
            elif not is_bean:
                entry["why"] = f"所在类不是 Bean：{why}"
                skipped.append(entry)
            elif default is not None:
                with_default.append(entry)
            else:
                required[key].append(entry)
    return required, skipped, with_default


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--all", action="store_true")
    ap.add_argument("--show-scan", action="store_true")
    ap.add_argument("--emit", metavar="PATH", help="生成配置草稿（.properties 格式）")
    args = ap.parse_args()

    packs, excluded = scan_config()
    if args.show_scan:
        print("扫描包（含递归得到的嵌套 @ComponentScan）：")
        for p in sorted(packs):
            print(f"  {p}")
        print(f"被 excludeFilters 排除的类：{sorted(excluded) or '（无）'}\n")

    required, skipped, with_default = scan_required(packs, excluded)
    provided = scan_provided()
    legacy = scan_legacy()

    # 值里还可能引用别的占位符（如 ${local.base.url}），递归补进「需要」集合
    needed, queue = {}, list(required)
    referenced = set()
    while queue:
        e = queue.pop()
        key = e["key"] if isinstance(e, dict) else e
        if key in needed:
            continue
        needed[key] = e
        base, _ = legacy_baseline(legacy, key)
        value = base[1] if base else None
        if not value:
            continue
        for ref in NESTED_RE.findall(value):
            rk, _ = split_placeholder(ref)
            if rk and rk not in needed:
                referenced.add(rk)
                queue.append(dict(key=rk, default=None, where=f"被 {key} 的值引用",
                                  why="", referenced_from=key))

    missing = {k: v for k, v in needed.items() if k not in provided}
    conflicts = {}
    for k in needed:
        _, values = legacy_baseline(legacy, k)
        if len(values) > 1:
            conflicts[k] = values

    print("=" * 78)
    print(f"【1】必需但新工程未提供（{len(missing)} 个）")
    print("    判据：注解存在（去注释）+ 不在 static 字段上 + 所在类是 Bean + 无默认值")
    print("=" * 78)
    for k in sorted(missing):
        e = missing[k]
        base, values = legacy_baseline(legacy, k)
        tag = "（由别的值引用）" if k in referenced else ""
        print(f"\n  ${k}{tag}")
        if base:
            print(f"      老工程里：{base[0]} → {base[1]!r}")
            if len(values) > 1:
                print(f"      ⚠️ 老工程多个 profile 值不同：{values}")
        else:
            print("      ⚠️ 老工程也没有定义（当初靠服务器上的外部配置/-D 参数注入，需部署时提供）")
        print(f"      引用处：{e['where'] if isinstance(e, dict) else e}")

    print(f"\n{'=' * 78}\n【2】有代码默认值、新工程未配（{len(with_default)} 个，需核对老工程值）\n{'=' * 78}")
    for e in sorted(with_default, key=lambda x: x["key"]):
        base, _ = legacy_baseline(legacy, e["key"])
        old = base[1] if base else "（老工程无）"
        flag = "  ⚠️ 与老值不同 → 行为静默改变" if base and base[1] != e["default"] else ""
        print(f"  {e['key']:<48} 代码默认={e['default']!r}  老值={old!r}{flag}")

    if args.all:
        print(f"\n{'=' * 78}\n【3】判为「无需配置」（{len(skipped)} 个，全是假阳性）\n{'=' * 78}")
        for e in sorted(skipped, key=lambda x: x["key"]):
            print(f"  {e['key']:<48} {e['why']}")
        print(f"\n{'=' * 78}\n【4】新工程已提供（{len(provided)} 个键中，被代码引用的）\n{'=' * 78}")
        for k in sorted(set(needed) & set(provided)):
            print(f"  {k:<48} {provided[k][0][0]} = {provided[k][0][1]}")

    if args.emit:
        lines = [
            "# 存量业务参数（从老工程各模块的 application*.properties 迁回；老工程见 AGENTS.md §3）",
            "# 生成脚本：tools/p2_audit_placeholders.py --emit",
            "# ⚠️ 值里的 ${...} 是**嵌套占位符**，由 Spring 在解析时再展开，不是笔误。",
            "# ⚠️ 凭据类键一律写成 ${环境变量名:}，真实值不进仓库（红线第 11 条）。",
            "",
        ]
        secrets = []
        for k in sorted(missing):
            base, values = legacy_baseline(legacy, k)
            env = re.sub(r'[^A-Za-z0-9]+', '_', k).upper()
            if any(h in k.lower() for h in SECRET_HINTS) or (base and _looks_secret(base[1])):
                lines.append(f"{k}=${{{env}:}}")
                secrets.append((k, base[1] if base else None, env))
            elif base:
                lines.append(f"{k}={base[1]}")
            else:
                lines.append(f"#{k}=   # 老工程未定义，需部署时按实际环境提供")
        with open(args.emit, "w", encoding="utf-8") as fh:
            fh.write("\n".join(lines) + "\n")
        print(f"\n已写出草稿：{args.emit}（{len(missing)} 个键）")
        if secrets:
            print("\n⚠️ 下列键的**老值被判定为凭据，未写入仓库**，改成了环境变量占位，"
                  "部署时必须注入（同时应把这些值加进「待轮换密钥」清单）：")
            for k, v, env in secrets:
                print(f"  {k:<40} ← 老值 {v!r}  环境变量 ${env}")

    print(f"\n合计：需要 {len(needed)} 个键（含被引用）；缺失 {len(missing)} 个；"
          f"有默认值待核对 {len(with_default)} 个；判为假阳性 {len(skipped)} 个。")
    return 1 if missing else 0


def _looks_secret(v):
    """凭据的启发式判据：长且含混合字符，且不像 URL / 路径 / 明文开关。"""
    if not v or v.startswith("${") or v.startswith("http") or "/" in v:
        return False
    if len(v) < 24:
        return False
    return bool(re.search(r'[a-z]', v) and re.search(r'[A-Z0-9]', v))


if __name__ == "__main__":
    sys.exit(main())