更新时间:2026-11-13 GMT+08:00
分享

写入与读取

读取Kafka写入Hive(创建Hive Function)

CREATE CATALOG lf_hive_catalog WITH (
  'type' = 'hive',
  'metastore'='lakeformation'
);

CREATE DATABASE IF NOT EXISTS lf_hive_catalog.hive_db
WITH (
  'location' = 'obs://fabricsql-hlt-test/aura-job/hive_db'
);

CREATE FUNCTION IF NOT EXISTS lf_hive_catalog.hive_db.hivefunction AS 'class_name' USING JAR 'obs://bucket/SimpleUdf.jar';

create table if not exists hive.hive_db.hive_table (
  user_id STRING,
  item_id STRING,
  cat_id STRING,
  ts timestamp(9),
  dy STRING,
  ho STRING,
  mi STRING,
  `value` String
) PARTITIONED BY (dy, ho, mi)with
  (
    'connector' = 'hive',
    'partition.time-extractor.timestamp-pattern' = '$dy $ho:$mi:00',
    'sink.partition-commit.trigger' = 'process-time',
    'sink.partition-commit.delay' = '0S',
    'sink.partition-commit.policy.kind' = 'metastore,success-file',
    'hive.storage.file-format'='avro'
  );

CREATE TABLE kafka_source (
  user_id STRING,
  item_id STRING,
  cat_id STRING,
  ts timestamp(9),
  dy STRING,
  ho STRING,
  mi STRING,
  `value` String
)
WITH
  (
  'connector' = 'kafka',
  'topic' = 'source_test',
  'properties.bootstrap.servers' = 'kafka-broker:9092',
  'properties.group.id' = 'source_test_consumer',
  'scan.startup.mode' = 'latest-offset',
  'format' = 'json'
  );

insert into lf_hive_catalog.hive_db.hive_table
select user_id,item_id,cat_id,ts,dy,ho,mi,lf_hive_catalog.hive_db.hivefunction(`value`) from kafka_source;

读取Hive写入Kafka

CREATE CATALOG lf_hive_catalog WITH (
  'type' = 'hive',
  'metastore'='lakeformation'
);

CREATE DATABASE IF NOT EXISTS lf_hive_catalog.hive_db
WITH (
  'location' = 'obs://fabricsql-hlt-test/aura-job/hive_db'
);

create table if not exists lf_hive_catalog.hive_db.hive_table (
  user_id STRING,
  item_id STRING,
  cat_id STRING,
  ts timestamp(9),
  dy STRING,
  ho STRING,
  mi STRING
) PARTITIONED BY (dy, ho, mi)with
  (
    'connector' = 'hive',
    'partition.time-extractor.timestamp-pattern' = '$dy $ho:$mi:00',
    'sink.partition-commit.trigger' = 'process-time',
    'sink.partition-commit.delay' = '0S',
    'sink.partition-commit.policy.kind' = 'metastore,success-file',
    'hive.storage.file-format'='avro'
  );

CREATE TABLE kafka_sink (
  user_id STRING,
  item_id STRING,
  cat_id STRING,
  ts timestamp(9),
  dy STRING,
  ho STRING,
  mi STRING
)
WITH
  (
   'connector' = 'kafka',
   'topic' = 'sink_test',
   'properties.bootstrap.servers' = 'kafka-broker:9092',
   'scan.startup.mode' = 'latest-offset',
   'format' = 'json'
  );

insert into kafka_sink
select user_id,item_id,cat_id,ts,dy,ho,mi from lf_hive_catalog.hive_db.hive_table;

相关文档