fix: 缩短标记唯一键避免超出 InnoDB 索引长度上限

This commit is contained in:
2026-10-09 21:00:33 +08:00
parent dbeafba23c
commit e5b3ef8eb6
2 changed files with 312 additions and 1 deletions
@@ -25,7 +25,14 @@ CREATE TABLE markers
deleted INT DEFAULT 0 NOT NULL COMMENT '逻辑删除符',
-- 同一对象上「同类 + 同名 + 同值」只保留一条(重复打标是幂等的)。
-- 值参与唯一键,因为系统标记是「名 = 值」成对表达的。
UNIQUE KEY uk_markers_object_marker (created_by, object_type, object_num, kind, marker_name, marker_value),
--
-- 注意:这里**不含 created_by**,两个原因:
-- 1. object_num 已经全局唯一(碎片自增 id 或任务编号),带上 created_by 是冗余;
-- 2. InnoDB 的索引键长度上限是 3072 字节(DYNAMIC 行格式);带上 created_by 后
-- 在 utf8mb4 下会达到 3620 字节,建表直接报 error 1071。
-- H2 不校验这个限制,所以只有真实 MySQL 才会暴露——改动索引时务必用
-- MigrationIndexKeyLengthTest 复算。
UNIQUE KEY uk_markers_object_marker (object_type, object_num, kind, marker_name, marker_value),
-- 按标记找对象(点标签即筛选)
KEY idx_markers_lookup (created_by, marker_name, object_type, object_num),
-- 列出某对象的全部标记
@@ -0,0 +1,304 @@
package com.guo.learningprogresstracker.migration;
import org.junit.jupiter.api.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
/**
* 迁移脚本的索引键长度守卫。
*
* <p>背景:InnoDB 对索引键长度有硬上限——DYNAMIC/COMPRESSED 行格式是 <b>3072 字节</b>,
* 超了直接报 error 1071 让建表/加索引失败。而本项目在 utf8mb4 下 VARCHAR(n) 要按 4n 字节算,
* 所以一个看起来人畜无害的「多列唯一键」很容易越界。</p>
*
* <p>危险之处在于<b>测试环境用 H2,它不校验这个限制</b>:迁移能在测试里跑通、单个 INSERT 也正常,
* 但一部署到真实 MySQL 就起不来。这条测试直接读迁移文件复算每个索引的键长度,
* 把这类问题挡在提交之前。</p>
*
* <p>新增/修改索引后若本测试失败:优先减少参与索引的列,或确认某些列在业务上是否真的需要进键。</p>
*/
class MigrationIndexKeyLengthTest {
/** InnoDB DYNAMIC 行格式下的索引键前缀长度上限(字节) */
private static final int MAX_KEY_BYTES = 3072;
/** utf8mb4 每字符最多 4 字节 */
private static final int UTF8MB4_BYTES_PER_CHAR = 4;
private static final Path MIGRATION_DIR = Path.of("src/main/resources/db/migration");
/** 形如 varchar(255) / tinyint(1) 的列声明 */
private static final Pattern WITH_LENGTH = Pattern.compile("^([a-z]+)\\((\\d+)(?:\\s*,\\s*\\d+)?\\)$");
/** 定长类型的字节数;VARCHAR/CHAR 按字符数 × 4 另算 */
private static final Map<String, Integer> FIXED_BYTES = Map.ofEntries(
Map.entry("tinyint", 1),
Map.entry("smallint", 2),
Map.entry("mediumint", 3),
Map.entry("int", 4),
Map.entry("integer", 4),
Map.entry("bigint", 8),
Map.entry("float", 4),
Map.entry("double", 8),
Map.entry("decimal", 8),
Map.entry("datetime", 8),
Map.entry("timestamp", 8),
Map.entry("date", 3),
Map.entry("time", 3),
Map.entry("year", 1));
private record IndexDef(String file, String table, String name, List<String> columns) {
}
@Test
void everyIndexFitsWithinInnoDbKeyLimit() throws Exception {
Map<String, Map<String, String>> tableColumns = new LinkedHashMap<>();
List<IndexDef> indexes = new ArrayList<>();
for (Path file : migrationFiles()) {
String sql = stripComments(Files.readString(file));
Parser parser = new Parser(file.getFileName().toString());
parser.parse(sql, tableColumns, indexes);
}
assertTrue(indexes.size() > 10, "应至少解析出若干索引,实际 " + indexes.size() + "(解析器是否失效?)");
List<String> violations = new ArrayList<>();
for (IndexDef index : indexes) {
Map<String, String> columns = tableColumns.get(index.table());
if (columns == null) {
violations.add(index.file() + " 的索引 " + index.name() + " 引用了未知表 " + index.table());
continue;
}
int total = 0;
List<String> unknown = new ArrayList<>();
for (String column : index.columns()) {
String declaredType = columns.get(column);
if (declaredType == null) {
unknown.add(column);
continue;
}
Integer bytes = bytesOfDeclared(declaredType);
if (bytes == null) {
// 只对「真的进了索引」的列报错:TEXT 必须先确认是否需要前缀索引
violations.add(index.file() + " 的索引 " + index.name() + " 引用了列 " + column
+ "(类型 " + declaredType + "),无法按定长计算键长");
continue;
}
total += bytes;
}
if (!unknown.isEmpty()) {
// 解析不到就报出来,避免新增列/写法变化时这条守卫静默失效
violations.add(index.file() + " 的索引 " + index.name() + " 引用了无法解析的列 " + unknown
+ "(表 " + index.table() + ")");
continue;
}
if (total > MAX_KEY_BYTES) {
violations.add(String.format("%s 的索引 %s 键长 %d 字节,超过 InnoDB 上限 %d 字节;列=%s",
index.file(), index.name(), total, MAX_KEY_BYTES, index.columns()));
}
}
if (!violations.isEmpty()) {
fail("迁移脚本里有索引键长度超限(真实 MySQL 会报 error 1071,H2 测不出来):\n - "
+ String.join("\n - ", violations));
}
}
/** 把列声明(如 varchar(255)、datetime)换算成索引键占用字节;无法定长计算的返回 null */
private static Integer bytesOfDeclared(String declaredType) {
String normalized = declaredType.toLowerCase(Locale.ROOT).strip();
Matcher withLength = WITH_LENGTH.matcher(normalized);
if (withLength.matches()) {
String base = withLength.group(1);
int length = Integer.parseInt(withLength.group(2));
if (base.equals("varchar") || base.equals("char")) {
return length * UTF8MB4_BYTES_PER_CHAR;
}
return FIXED_BYTES.get(base);
}
return FIXED_BYTES.get(normalized);
}
private static List<Path> migrationFiles() throws Exception {
try (Stream<Path> files = Files.list(MIGRATION_DIR)) {
return files.filter(path -> path.getFileName().toString().endsWith(".sql"))
.sorted(Comparator.comparing(path -> path.getFileName().toString()))
.toList();
}
}
/** 去掉整行注释,避免注释里的示例被当成 DDL */
private static String stripComments(String sql) {
StringBuilder result = new StringBuilder();
for (String line : sql.split("\r?\n")) {
if (line.trim().startsWith("--")) {
continue;
}
result.append(line).append('\n');
}
return result.toString();
}
/** 极简 DDL 解析器:只覆盖本仓库实际用到的写法,遇到不认识的列会让测试失败而不是静默放过 */
private static final class Parser {
private static final Pattern CREATE_TABLE =
Pattern.compile("(?i)^\\s*create\\s+table\\s+(?:if\\s+not\\s+exists\\s+)?(\\w+)\\s*\\(?\\s*$");
private static final Pattern COLUMN =
Pattern.compile("(?i)^\\s*(\\w+)\\s+([a-z]+)\\s*(?:\\(\\s*(\\d+)(?:\\s*,\\s*\\d+)?\\s*\\))?");
private static final Pattern ALTER_TABLE = Pattern.compile("(?i)^\\s*alter\\s+table\\s+(\\w+)\\s+(.*)$");
private static final Pattern ADD_OR_MODIFY_COLUMN =
Pattern.compile("(?i)^(?:add|modify)\\s+column\\s+(\\w+)\\s+([a-z]+)\\s*(?:\\(\\s*(\\d+)(?:\\s*,\\s*\\d+)?\\s*\\))?");
private static final Pattern RENAME_COLUMN =
Pattern.compile("(?i)^rename\\s+column\\s+(\\w+)\\s+to\\s+(\\w+)");
private static final Pattern INLINE_KEY =
Pattern.compile("(?i)^\\s*(?:unique\\s+)?(?:key|index)\\s+(\\w+)\\s*\\(([^)]*)\\)");
private static final Pattern CREATE_INDEX =
Pattern.compile("(?i)^\\s*create\\s+(?:unique\\s+)?index\\s+(\\w+)\\s+on\\s+(\\w+)\\s*\\(([^)]*)\\)");
private static final Pattern ADD_UNIQUE_CONSTRAINT =
Pattern.compile("(?i)^add\\s+constraint\\s+(\\w+)\\s+unique\\s*\\(([^)]*)\\)");
private final String fileName;
private String currentTable;
private boolean insideTableBody;
Parser(String fileName) {
this.fileName = fileName;
}
void parse(String sql, Map<String, Map<String, String>> tableColumns, List<IndexDef> indexes) {
for (String rawLine : sql.split("\r?\n")) {
String line = rawLine.strip();
if (line.isEmpty()) {
continue;
}
Matcher createTable = CREATE_TABLE.matcher(line);
if (createTable.matches()) {
currentTable = createTable.group(1).toLowerCase(Locale.ROOT);
insideTableBody = true;
tableColumns.computeIfAbsent(currentTable, key -> new LinkedHashMap<>());
continue;
}
if (insideTableBody) {
if (line.startsWith(")")) {
insideTableBody = false;
continue;
}
if (line.startsWith("(")) {
continue;
}
Matcher inlineKey = INLINE_KEY.matcher(line);
if (inlineKey.find()) {
indexes.add(new IndexDef(fileName, currentTable, inlineKey.group(1),
splitColumns(inlineKey.group(2))));
continue;
}
if (isConstraintKeyword(line)) {
continue;
}
Matcher column = COLUMN.matcher(line);
if (column.find()) {
String name = column.group(1).toLowerCase(Locale.ROOT);
if (isTypeKeyword(name)) {
continue;
}
// 只记声明,等索引真正引用到它时再算字节(避免对没进索引的 TEXT 列误报)
tableColumns.get(currentTable).put(name, declaredType(column.group(2), column.group(3)));
}
continue;
}
Matcher createIndex = CREATE_INDEX.matcher(line);
if (createIndex.matches()) {
indexes.add(new IndexDef(fileName, createIndex.group(2).toLowerCase(Locale.ROOT),
createIndex.group(1), splitColumns(createIndex.group(3))));
continue;
}
Matcher alterTable = ALTER_TABLE.matcher(line);
if (!alterTable.matches()) {
continue;
}
String table = alterTable.group(1).toLowerCase(Locale.ROOT);
String rest = alterTable.group(2).strip();
Map<String, String> columns = tableColumns.computeIfAbsent(table, key -> new LinkedHashMap<>());
Matcher addUnique = ADD_UNIQUE_CONSTRAINT.matcher(rest);
if (addUnique.find()) {
indexes.add(new IndexDef(fileName, table, addUnique.group(1), splitColumns(addUnique.group(2))));
continue;
}
Matcher rename = RENAME_COLUMN.matcher(rest);
if (rename.find()) {
String from = rename.group(1).toLowerCase(Locale.ROOT);
String to = rename.group(2).toLowerCase(Locale.ROOT);
String declared = columns.remove(from);
if (declared != null) {
columns.put(to, declared);
}
continue;
}
Matcher column = ADD_OR_MODIFY_COLUMN.matcher(rest);
if (column.find()) {
String name = column.group(1).toLowerCase(Locale.ROOT);
columns.put(name, declaredType(column.group(2), column.group(3)));
}
}
}
private static boolean isConstraintKeyword(String line) {
String lower = line.toLowerCase(Locale.ROOT);
return lower.startsWith("primary ")
|| lower.startsWith("foreign ")
|| lower.startsWith("constraint ")
|| lower.startsWith("check ");
}
private static boolean isTypeKeyword(String name) {
return switch (name) {
case "primary", "foreign", "constraint", "check", "unique", "key", "index" -> true;
default -> false;
};
}
private static List<String> splitColumns(String raw) {
List<String> columns = new ArrayList<>();
for (String part : raw.split(",")) {
String name = part.strip().toLowerCase(Locale.ROOT);
// 去掉可能的后缀(如 name(50) 前缀索引、ASC/DESC)
int paren = name.indexOf('(');
if (paren > 0) {
name = name.substring(0, paren);
}
name = name.replaceAll("\\s+(asc|desc)$", "").strip();
if (!name.isEmpty()) {
columns.add(name);
}
}
return columns;
}
private static String declaredType(String type, String length) {
String base = type.toLowerCase(Locale.ROOT);
return length == null ? base : base + "(" + length + ")";
}
}
}