Liquid Clustering使用方式

Liquid Clustering配置

Liquid Clustering通过Iceberg表属性进行配置。

CREATE TABLE events (
    event_id BIGINT,
    user_id BIGINT,
    event_type STRING,
    event_time TIMESTAMP,
    payload STRING
)
USING iceberg
TBLPROPERTIES (
    'format-version' = '2',
    'write.liquid-clustering.columns' = 'user_id,event_time'
);

指定自定义Cube大小

CREATE TABLE logs (
    log_id BIGINT,
    service_name STRING,
    log_level STRING,
    message STRING,
    timestamp TIMESTAMP
)
USING iceberg
TBLPROPERTIES (
    'format-version' = '2',
    'write.liquid-clustering.columns' = 'service_name,timestamp',
    'write.liquid-clustering.target-cube-size-bytes' = '21474836480',
    'write.liquid-clustering.min-cube-size-bytes' = '5368709120'
);

修改Liquid Clustering配置

-- 修改聚类列（仅支持添加，不支持删除已有聚类列）
ALTER TABLE events SET TBLPROPERTIES ('write.liquid-clustering.columns' = 'user_id,event_time,event_type');

-- 调整目标Cube大小
ALTER TABLE events SET TBLPROPERTIES ('write.liquid-clustering.target-cube-size-bytes' = '32212254720');

-- 调整最小Cube阈值
ALTER TABLE events SET TBLPROPERTIES ('write.liquid-clustering.min-cube-size-bytes' = '10737418240');

触发Cube重写

-- 重写所有标记为partial的Cube
CALL spark_catalog.system.rewrite_data_files(
    table => 'db.events',
    options => map('rewrite-all', 'false')
);

-- 重写所有Cube（包括健康的Cube）
CALL spark_catalog.system.rewrite_data_files(
    table => 'db.events',
    options => map('rewrite-all', 'true')
);