【详述】kafka导入数据出现PAUSE
【业务影响】不能稳定导入数据
【StarRocks版本】3.0.3
【集群规模】例如：3fe+3be（fe与be混部）
【机器信息】8/32g/3台
【表模型】：明细模型
【导入或者导出方式】：kafka
【联系方式】站内邮件
【附件】
CREATE ROUTINE LOAD olap.click_data_load ON click_data
PROPERTIES
(
“desired_concurrent_number” = “3”,
“max_error_number” = “10000”,
“format” = “json”
)
FROM KAFKA
(
“kafka_broker_list” = “172.18.0.:19092,172.18.0.:19092, 172.18.0.*:19092”,
“kafka_topic” = “starrocks”,
“kafka_partitions” = “0,1,2,3,4,5”,
“property.kafka_default_offsets” = “OFFSET_END”,
“property.group.id” =“click_data_load_group”
);
建表：
CREATE TABLE olap.click_data
(
doc_id VARCHAR(200) NOT NULL,
day VARCHAR(8) NOT NULL,
aff_id VARCHAR(10),
offer_id VARCHAR(10),
app_id VARCHAR(100),
country VARCHAR(10),
platform VARCHAR(7),
device_id VARCHAR(64),
media VARCHAR(5),
media_source VARCHAR(100),
model VARCHAR(100),
make VARCHAR(100),
bundle VARCHAR(100),
cat VARCHAR(1000),
click_id VARCHAR(2048),
time DATETIME NOT NULL,
INDEX click_data_day (day) USING BITMAP ,
INDEX click_data_aff_id (aff_id) USING BITMAP ,
INDEX click_data_offer_id (offer_id) USING BITMAP ,
INDEX click_data_app_id (app_id) USING BITMAP ,
INDEX click_data_country (country) USING BITMAP,
INDEX click_data_platform (platform) USING BITMAP,
INDEX click_data_media (media) USING BITMAP,
INDEX click_data_media_source (media_source) USING BITMAP,
INDEX click_data_model (model) USING BITMAP,
INDEX click_data_make (make) USING BITMAP,
INDEX click_data_bundle (bundle) USING BITMAP,
INDEX click_data_cat (cat) USING BITMAP,
INDEX time (time) USING BITMAP
)
DUPLICATE KEY(doc_id,day,aff_id,offer_id,app_id,country,platform)
PARTITION BY (day)
DISTRIBUTED BY HASH(doc_id, day,offer_id,app_id,aff_id,country,platform,device_id)BUCKETS 48
PROPERTIES (
“replication_num” = “2”,
“enable_persistent_index” = “true”
)
ErrorLogurl打开看到：
Error: The row create partition failed since Runtime error: automatic create partition failed. error:Duplicate values (“20230816”) … Row: [‘tN2KXTR93y’, ‘20230816’, ‘221’, ‘1003419’, ‘com.zhiliaoapp.musically’, ‘us’, ‘1’, 'dfecb396-47a1-4b72-ae5d-b2
27560f1a16’, ‘12’, NULL, ‘KOB09’, ‘’, ‘gocrsmartnews.android’, ‘IAB12 IAB1’, '31323202744e324b585452393379024b4f422d4c303902026a702e676f63726f2e736d6172746e6577732e616e64726f69640249414231322049
414231__6a7d7__v3304711059067d4240376f24cba9a80d1f8781c4ab7’, 2023-08-16 00:00:01]
插入语句： INSERT
INTO
olap.click_data (doc_id,
day,
aff_id,
offer_id,
app_id,
country,
platform,
device_id,
media,
media_source,
model,
make,
bundle,
cat,
click_id,
time)
VALUES (‘tN2KXTR93y’,
‘20230817’,
‘221’,
‘10034419’,
‘com.zhiliaoapp.musically’,
‘us’,
‘1’,
‘dfecb396-47a1-4b72-ae5d-b227560f1a16’,
‘122’,
NULL,
‘KOB-L09’,
‘’,
‘jp.gocro.smartnews.android’,
‘IAB12 IAB1’,
‘000221__10034419__202308160000606855550209920____31323202744e324b585452393379024b4f422d4c303902026a702e676f63726f2e736d6172746e6577732e616e64726f69640249414231322049414231__6a7d7__v3304711059067d4240376f24cba9a80d1f8781c4ab7’,
‘2023-08-16 00:00:01’);