fix: 缩短标记唯一键避免超出 InnoDB 索引长度上限
This commit is contained in:
@@ -25,7 +25,14 @@ CREATE TABLE markers
|
||||
deleted INT DEFAULT 0 NOT NULL COMMENT '逻辑删除符',
|
||||
-- 同一对象上「同类 + 同名 + 同值」只保留一条(重复打标是幂等的)。
|
||||
-- 值参与唯一键,因为系统标记是「名 = 值」成对表达的。
|
||||
UNIQUE KEY uk_markers_object_marker (created_by, object_type, object_num, kind, marker_name, marker_value),
|
||||
--
|
||||
-- 注意:这里**不含 created_by**,两个原因:
|
||||
-- 1. object_num 已经全局唯一(碎片自增 id 或任务编号),带上 created_by 是冗余;
|
||||
-- 2. InnoDB 的索引键长度上限是 3072 字节(DYNAMIC 行格式);带上 created_by 后
|
||||
-- 在 utf8mb4 下会达到 3620 字节,建表直接报 error 1071。
|
||||
-- H2 不校验这个限制,所以只有真实 MySQL 才会暴露——改动索引时务必用
|
||||
-- MigrationIndexKeyLengthTest 复算。
|
||||
UNIQUE KEY uk_markers_object_marker (object_type, object_num, kind, marker_name, marker_value),
|
||||
-- 按标记找对象(点标签即筛选)
|
||||
KEY idx_markers_lookup (created_by, marker_name, object_type, object_num),
|
||||
-- 列出某对象的全部标记
|
||||
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
package com.guo.learningprogresstracker.migration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
/**
|
||||
* 迁移脚本的索引键长度守卫。
|
||||
*
|
||||
* <p>背景:InnoDB 对索引键长度有硬上限——DYNAMIC/COMPRESSED 行格式是 <b>3072 字节</b>,
|
||||
* 超了直接报 error 1071 让建表/加索引失败。而本项目在 utf8mb4 下 VARCHAR(n) 要按 4n 字节算,
|
||||
* 所以一个看起来人畜无害的「多列唯一键」很容易越界。</p>
|
||||
*
|
||||
* <p>危险之处在于<b>测试环境用 H2,它不校验这个限制</b>:迁移能在测试里跑通、单个 INSERT 也正常,
|
||||
* 但一部署到真实 MySQL 就起不来。这条测试直接读迁移文件复算每个索引的键长度,
|
||||
* 把这类问题挡在提交之前。</p>
|
||||
*
|
||||
* <p>新增/修改索引后若本测试失败:优先减少参与索引的列,或确认某些列在业务上是否真的需要进键。</p>
|
||||
*/
|
||||
class MigrationIndexKeyLengthTest {
|
||||
|
||||
/** InnoDB DYNAMIC 行格式下的索引键前缀长度上限(字节) */
|
||||
private static final int MAX_KEY_BYTES = 3072;
|
||||
|
||||
/** utf8mb4 每字符最多 4 字节 */
|
||||
private static final int UTF8MB4_BYTES_PER_CHAR = 4;
|
||||
|
||||
private static final Path MIGRATION_DIR = Path.of("src/main/resources/db/migration");
|
||||
|
||||
/** 形如 varchar(255) / tinyint(1) 的列声明 */
|
||||
private static final Pattern WITH_LENGTH = Pattern.compile("^([a-z]+)\\((\\d+)(?:\\s*,\\s*\\d+)?\\)$");
|
||||
|
||||
/** 定长类型的字节数;VARCHAR/CHAR 按字符数 × 4 另算 */
|
||||
private static final Map<String, Integer> FIXED_BYTES = Map.ofEntries(
|
||||
Map.entry("tinyint", 1),
|
||||
Map.entry("smallint", 2),
|
||||
Map.entry("mediumint", 3),
|
||||
Map.entry("int", 4),
|
||||
Map.entry("integer", 4),
|
||||
Map.entry("bigint", 8),
|
||||
Map.entry("float", 4),
|
||||
Map.entry("double", 8),
|
||||
Map.entry("decimal", 8),
|
||||
Map.entry("datetime", 8),
|
||||
Map.entry("timestamp", 8),
|
||||
Map.entry("date", 3),
|
||||
Map.entry("time", 3),
|
||||
Map.entry("year", 1));
|
||||
|
||||
private record IndexDef(String file, String table, String name, List<String> columns) {
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyIndexFitsWithinInnoDbKeyLimit() throws Exception {
|
||||
Map<String, Map<String, String>> tableColumns = new LinkedHashMap<>();
|
||||
List<IndexDef> indexes = new ArrayList<>();
|
||||
|
||||
for (Path file : migrationFiles()) {
|
||||
String sql = stripComments(Files.readString(file));
|
||||
Parser parser = new Parser(file.getFileName().toString());
|
||||
parser.parse(sql, tableColumns, indexes);
|
||||
}
|
||||
|
||||
assertTrue(indexes.size() > 10, "应至少解析出若干索引,实际 " + indexes.size() + "(解析器是否失效?)");
|
||||
|
||||
List<String> violations = new ArrayList<>();
|
||||
for (IndexDef index : indexes) {
|
||||
Map<String, String> columns = tableColumns.get(index.table());
|
||||
if (columns == null) {
|
||||
violations.add(index.file() + " 的索引 " + index.name() + " 引用了未知表 " + index.table());
|
||||
continue;
|
||||
}
|
||||
int total = 0;
|
||||
List<String> unknown = new ArrayList<>();
|
||||
for (String column : index.columns()) {
|
||||
String declaredType = columns.get(column);
|
||||
if (declaredType == null) {
|
||||
unknown.add(column);
|
||||
continue;
|
||||
}
|
||||
Integer bytes = bytesOfDeclared(declaredType);
|
||||
if (bytes == null) {
|
||||
// 只对「真的进了索引」的列报错:TEXT 必须先确认是否需要前缀索引
|
||||
violations.add(index.file() + " 的索引 " + index.name() + " 引用了列 " + column
|
||||
+ "(类型 " + declaredType + "),无法按定长计算键长");
|
||||
continue;
|
||||
}
|
||||
total += bytes;
|
||||
}
|
||||
if (!unknown.isEmpty()) {
|
||||
// 解析不到就报出来,避免新增列/写法变化时这条守卫静默失效
|
||||
violations.add(index.file() + " 的索引 " + index.name() + " 引用了无法解析的列 " + unknown
|
||||
+ "(表 " + index.table() + ")");
|
||||
continue;
|
||||
}
|
||||
if (total > MAX_KEY_BYTES) {
|
||||
violations.add(String.format("%s 的索引 %s 键长 %d 字节,超过 InnoDB 上限 %d 字节;列=%s",
|
||||
index.file(), index.name(), total, MAX_KEY_BYTES, index.columns()));
|
||||
}
|
||||
}
|
||||
|
||||
if (!violations.isEmpty()) {
|
||||
fail("迁移脚本里有索引键长度超限(真实 MySQL 会报 error 1071,H2 测不出来):\n - "
|
||||
+ String.join("\n - ", violations));
|
||||
}
|
||||
}
|
||||
|
||||
/** 把列声明(如 varchar(255)、datetime)换算成索引键占用字节;无法定长计算的返回 null */
|
||||
private static Integer bytesOfDeclared(String declaredType) {
|
||||
String normalized = declaredType.toLowerCase(Locale.ROOT).strip();
|
||||
Matcher withLength = WITH_LENGTH.matcher(normalized);
|
||||
if (withLength.matches()) {
|
||||
String base = withLength.group(1);
|
||||
int length = Integer.parseInt(withLength.group(2));
|
||||
if (base.equals("varchar") || base.equals("char")) {
|
||||
return length * UTF8MB4_BYTES_PER_CHAR;
|
||||
}
|
||||
return FIXED_BYTES.get(base);
|
||||
}
|
||||
return FIXED_BYTES.get(normalized);
|
||||
}
|
||||
|
||||
private static List<Path> migrationFiles() throws Exception {
|
||||
try (Stream<Path> files = Files.list(MIGRATION_DIR)) {
|
||||
return files.filter(path -> path.getFileName().toString().endsWith(".sql"))
|
||||
.sorted(Comparator.comparing(path -> path.getFileName().toString()))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
/** 去掉整行注释,避免注释里的示例被当成 DDL */
|
||||
private static String stripComments(String sql) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (String line : sql.split("\r?\n")) {
|
||||
if (line.trim().startsWith("--")) {
|
||||
continue;
|
||||
}
|
||||
result.append(line).append('\n');
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/** 极简 DDL 解析器:只覆盖本仓库实际用到的写法,遇到不认识的列会让测试失败而不是静默放过 */
|
||||
private static final class Parser {
|
||||
|
||||
private static final Pattern CREATE_TABLE =
|
||||
Pattern.compile("(?i)^\\s*create\\s+table\\s+(?:if\\s+not\\s+exists\\s+)?(\\w+)\\s*\\(?\\s*$");
|
||||
private static final Pattern COLUMN =
|
||||
Pattern.compile("(?i)^\\s*(\\w+)\\s+([a-z]+)\\s*(?:\\(\\s*(\\d+)(?:\\s*,\\s*\\d+)?\\s*\\))?");
|
||||
private static final Pattern ALTER_TABLE = Pattern.compile("(?i)^\\s*alter\\s+table\\s+(\\w+)\\s+(.*)$");
|
||||
private static final Pattern ADD_OR_MODIFY_COLUMN =
|
||||
Pattern.compile("(?i)^(?:add|modify)\\s+column\\s+(\\w+)\\s+([a-z]+)\\s*(?:\\(\\s*(\\d+)(?:\\s*,\\s*\\d+)?\\s*\\))?");
|
||||
private static final Pattern RENAME_COLUMN =
|
||||
Pattern.compile("(?i)^rename\\s+column\\s+(\\w+)\\s+to\\s+(\\w+)");
|
||||
private static final Pattern INLINE_KEY =
|
||||
Pattern.compile("(?i)^\\s*(?:unique\\s+)?(?:key|index)\\s+(\\w+)\\s*\\(([^)]*)\\)");
|
||||
private static final Pattern CREATE_INDEX =
|
||||
Pattern.compile("(?i)^\\s*create\\s+(?:unique\\s+)?index\\s+(\\w+)\\s+on\\s+(\\w+)\\s*\\(([^)]*)\\)");
|
||||
private static final Pattern ADD_UNIQUE_CONSTRAINT =
|
||||
Pattern.compile("(?i)^add\\s+constraint\\s+(\\w+)\\s+unique\\s*\\(([^)]*)\\)");
|
||||
|
||||
private final String fileName;
|
||||
private String currentTable;
|
||||
private boolean insideTableBody;
|
||||
|
||||
Parser(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
void parse(String sql, Map<String, Map<String, String>> tableColumns, List<IndexDef> indexes) {
|
||||
for (String rawLine : sql.split("\r?\n")) {
|
||||
String line = rawLine.strip();
|
||||
if (line.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher createTable = CREATE_TABLE.matcher(line);
|
||||
if (createTable.matches()) {
|
||||
currentTable = createTable.group(1).toLowerCase(Locale.ROOT);
|
||||
insideTableBody = true;
|
||||
tableColumns.computeIfAbsent(currentTable, key -> new LinkedHashMap<>());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (insideTableBody) {
|
||||
if (line.startsWith(")")) {
|
||||
insideTableBody = false;
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("(")) {
|
||||
continue;
|
||||
}
|
||||
Matcher inlineKey = INLINE_KEY.matcher(line);
|
||||
if (inlineKey.find()) {
|
||||
indexes.add(new IndexDef(fileName, currentTable, inlineKey.group(1),
|
||||
splitColumns(inlineKey.group(2))));
|
||||
continue;
|
||||
}
|
||||
if (isConstraintKeyword(line)) {
|
||||
continue;
|
||||
}
|
||||
Matcher column = COLUMN.matcher(line);
|
||||
if (column.find()) {
|
||||
String name = column.group(1).toLowerCase(Locale.ROOT);
|
||||
if (isTypeKeyword(name)) {
|
||||
continue;
|
||||
}
|
||||
// 只记声明,等索引真正引用到它时再算字节(避免对没进索引的 TEXT 列误报)
|
||||
tableColumns.get(currentTable).put(name, declaredType(column.group(2), column.group(3)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher createIndex = CREATE_INDEX.matcher(line);
|
||||
if (createIndex.matches()) {
|
||||
indexes.add(new IndexDef(fileName, createIndex.group(2).toLowerCase(Locale.ROOT),
|
||||
createIndex.group(1), splitColumns(createIndex.group(3))));
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher alterTable = ALTER_TABLE.matcher(line);
|
||||
if (!alterTable.matches()) {
|
||||
continue;
|
||||
}
|
||||
String table = alterTable.group(1).toLowerCase(Locale.ROOT);
|
||||
String rest = alterTable.group(2).strip();
|
||||
Map<String, String> columns = tableColumns.computeIfAbsent(table, key -> new LinkedHashMap<>());
|
||||
|
||||
Matcher addUnique = ADD_UNIQUE_CONSTRAINT.matcher(rest);
|
||||
if (addUnique.find()) {
|
||||
indexes.add(new IndexDef(fileName, table, addUnique.group(1), splitColumns(addUnique.group(2))));
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher rename = RENAME_COLUMN.matcher(rest);
|
||||
if (rename.find()) {
|
||||
String from = rename.group(1).toLowerCase(Locale.ROOT);
|
||||
String to = rename.group(2).toLowerCase(Locale.ROOT);
|
||||
String declared = columns.remove(from);
|
||||
if (declared != null) {
|
||||
columns.put(to, declared);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher column = ADD_OR_MODIFY_COLUMN.matcher(rest);
|
||||
if (column.find()) {
|
||||
String name = column.group(1).toLowerCase(Locale.ROOT);
|
||||
columns.put(name, declaredType(column.group(2), column.group(3)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean isConstraintKeyword(String line) {
|
||||
String lower = line.toLowerCase(Locale.ROOT);
|
||||
return lower.startsWith("primary ")
|
||||
|| lower.startsWith("foreign ")
|
||||
|| lower.startsWith("constraint ")
|
||||
|| lower.startsWith("check ");
|
||||
}
|
||||
|
||||
private static boolean isTypeKeyword(String name) {
|
||||
return switch (name) {
|
||||
case "primary", "foreign", "constraint", "check", "unique", "key", "index" -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private static List<String> splitColumns(String raw) {
|
||||
List<String> columns = new ArrayList<>();
|
||||
for (String part : raw.split(",")) {
|
||||
String name = part.strip().toLowerCase(Locale.ROOT);
|
||||
// 去掉可能的后缀(如 name(50) 前缀索引、ASC/DESC)
|
||||
int paren = name.indexOf('(');
|
||||
if (paren > 0) {
|
||||
name = name.substring(0, paren);
|
||||
}
|
||||
name = name.replaceAll("\\s+(asc|desc)$", "").strip();
|
||||
if (!name.isEmpty()) {
|
||||
columns.add(name);
|
||||
}
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
private static String declaredType(String type, String length) {
|
||||
String base = type.toLowerCase(Locale.ROOT);
|
||||
return length == null ? base : base + "(" + length + ")";
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user