broker load拉取hdfs数据经常中途停住 然后一直不动

[a1265262132]

【详述】我用broker load方式同步hdfs上6亿+条数据，show load 查看scan row 一直上涨，但是同步到4亿多条后，show load查看，scan row停止，一直停在那里不动，直到任务超时，重复多次依旧这样。同步几千万的任务也会出现这种问题。
【建表语句】
CREATE TABLE ods_g3_dbo_subfhd_bak (
subbh varchar(120) NOT NULL COMMENT “”,
dh varchar(600) NOT NULL COMMENT “”,
kdrq date NOT NULL COMMENT “”,
fid decimal64(10, 0) NULL COMMENT “”,
fyear decimal64(10, 0) NULL COMMENT “”,
djlx decimal64(10, 0) NULL COMMENT “”,
fperiod decimal64(10, 0) NULL COMMENT “”,
ftype decimal64(10, 0) NULL COMMENT “”,
jzbz decimal64(1, 0) NULL COMMENT “”,
shbz decimal64(1, 0) NULL COMMENT “”,
lsh varchar(120) NULL COMMENT “”,
lb varchar(120) NULL COMMENT “”,
hh varchar(120) NULL COMMENT “”,
zzsl decimal64(18, 0) NULL COMMENT “”,
sl decimal64(18, 6) NULL COMMENT “”,
zk decimal64(18, 2) NULL COMMENT “”,
dj decimal64(18, 6) NULL COMMENT “”,
lsj decimal64(18, 6) NULL COMMENT “”,
sj decimal64(18, 6) NULL COMMENT “”,
je decimal64(18, 2) NULL COMMENT “”,
se decimal64(18, 2) NULL COMMENT “”,
jshj decimal64(18, 2) NULL COMMENT “”,
cbj decimal64(18, 8) NULL COMMENT “”,
cbe decimal64(18, 2) NULL COMMENT “”,
ml decimal64(18, 2) NULL COMMENT “”,
lsze decimal64(18, 2) NULL COMMENT “”,
cj decimal64(18, 2) NULL COMMENT “”,
xsr varchar(200) NULL COMMENT “”,
bz varchar(400) NULL COMMENT “”,
czry varchar(120) NULL COMMENT “”,
ph1 varchar(120) NULL COMMENT “”,
ph2 varchar(120) NULL COMMENT “”,
ph3_sl decimal64(18, 6) NULL COMMENT “”,
ph1_xq date NULL COMMENT “”,
fvchnum decimal64(10, 0) NULL COMMENT “”,
fgroup varchar(120) NULL COMMENT “”,
fvch decimal64(1, 0) NULL COMMENT “”,
stockid varchar(120) NULL COMMENT “”,
bm varchar(120) NULL COMMENT “”,
yhkh varchar(200) NULL COMMENT “”,
pcdh varchar(1200) NULL COMMENT “”,
fpzdate date NULL COMMENT “”,
fpzperiod decimal64(10, 0) NULL COMMENT “”,
fpzyear decimal64(10, 0) NULL COMMENT “”,
vchoperation varchar(120) NULL COMMENT “”,
fvchid varchar(120) NULL COMMENT “”,
jzry varchar(200) NULL COMMENT “”,
commissionrate decimal64(18, 6) NULL COMMENT “”,
scrq datetime NULL COMMENT “”,
batchprice decimal64(18, 6) NULL COMMENT “”,
billtime varchar(40) NULL COMMENT “”,
jzrq datetime NULL COMMENT “”,
balancebaseprice decimal64(18, 6) NULL COMMENT “”,
integralvalue decimal64(18, 2) NULL COMMENT “”,
posguid varchar(100) NULL COMMENT “”,
dealguid varchar(100) NULL COMMENT “”,
daliylogdate date NULL COMMENT “”,
customerinfo varchar(200) NULL COMMENT “”,
zblastpurchaseprice decimal64(18, 6) NULL COMMENT “”,
zbjhdj decimal64(18, 6) NULL COMMENT “”,
headbatchnumber varchar(100) NULL COMMENT “”,
suppliercode varchar(100) NULL COMMENT “”,
cxfaid varchar(100) NULL COMMENT “”,
pfml decimal64(18, 6) NULL COMMENT “”,
hsml decimal64(18, 6) NULL COMMENT “”,
rightoptcode varchar(100) NULL COMMENT “”,
workshiftid varchar(100) NULL COMMENT “”,
tcfs varchar(20) NULL COMMENT “”,
cxfamxid varchar(100) NULL COMMENT “”,
kh varchar(120) NULL COMMENT “”,
policytype decimal128(27, 0) NULL COMMENT “”,
fsfcj decimal64(1, 0) NULL COMMENT “”,
cdbz decimal64(1, 0) NULL COMMENT “”,
personname varchar(60) NULL COMMENT “”,
idcardno varchar(100) NULL COMMENT “”,
boxid varchar(100) NULL COMMENT “”,
checknum decimal64(18, 1) NULL COMMENT “”,
isrecheck decimal64(1, 0) NULL COMMENT “”,
recheckdate date NULL COMMENT “”,
recheckoperatorguid varchar(100) NULL COMMENT “”,
tjbh varchar(200) NULL COMMENT “”,
auditingname varchar(100) NULL COMMENT “”,
patientname varchar(100) NULL COMMENT “”,
doctorname varchar(100) NULL COMMENT “”,
doctorcode varchar(100) NULL COMMENT “”,
clname varchar(100) NULL COMMENT “”,
pharmacistname varchar(100) NULL COMMENT “”,
appdetailguid varchar(100) NULL COMMENT “”,
appguid varchar(100) NULL COMMENT “”,
address varchar(400) NULL COMMENT “”,
bh varchar(120) NULL COMMENT “”,
branchguid varchar(60) NULL COMMENT “”,
bs decimal64(10, 0) NULL COMMENT “”,
cd varchar(600) NULL COMMENT “”,
clfcr varchar(40) NULL COMMENT “”,
clfhr varchar(40) NULL COMMENT “”,
c_ftbz decimal64(18, 6) NULL COMMENT “”,
c_ftje decimal64(18, 6) NULL COMMENT “”,
c_ftjs decimal64(18, 6) NULL COMMENT “”,
c_ftml decimal64(18, 6) NULL COMMENT “”,
c_ftse decimal64(18, 6) NULL COMMENT “”,
c_zsfy decimal64(1, 0) NULL COMMENT “”,
dw varchar(120) NULL COMMENT “”,
dwmc varchar(400) NULL COMMENT “”,
dydh decimal64(10, 0) NULL COMMENT “”,
falreadyconsignnum decimal64(18, 6) NULL COMMENT “”,
fconsignbillmark decimal64(1, 0) NULL COMMENT “”,
fhbz decimal64(10, 0) NULL COMMENT “”,
fhrq date NULL COMMENT “”,
fhsl decimal64(18, 6) NULL COMMENT “”,
fhsysl decimal64(18, 6) NULL COMMENT “”,
fk_tag decimal64(10, 0) NULL COMMENT “”,
fmanyconsignmark decimal64(1, 0) NULL COMMENT “”,
fp_tag decimal64(10, 0) NULL COMMENT “”,
fremainmcnum decimal64(18, 6) NULL COMMENT “”,
fwithdrawgoodsbillid varchar(120) NULL COMMENT “”,
fwithdrawgoodsmark decimal64(1, 0) NULL COMMENT “”,
gcbh varchar(120) NULL COMMENT “”,
gg varchar(320) NULL COMMENT “”,
gspxsy varchar(100) NULL COMMENT “”,
hksl decimal64(18, 6) NULL COMMENT “”,
hkwb decimal64(1, 0) NULL COMMENT “”,
hk_date date NULL COMMENT “”,
hk_tag decimal64(1, 0) NULL COMMENT “”,
htbh varchar(120) NULL COMMENT “”,
hyj decimal64(18, 6) NULL COMMENT “”,
isauditing decimal64(1, 0) NULL COMMENT “”,
isqq varchar(100) NULL COMMENT “”,
istj varchar(100) NULL COMMENT “”,
istran_crm decimal64(1, 0) NULL COMMENT “”,
jj decimal64(18, 6) NULL COMMENT “”,
js decimal64(18, 6) NULL COMMENT “”,
jsfs decimal64(10, 0) NULL COMMENT “”,
jsje decimal64(18, 2) NULL COMMENT “”,
jsjs decimal64(18, 2) NULL COMMENT “”,
jsse decimal64(18, 2) NULL COMMENT “”,
jssl decimal64(18, 6) NULL COMMENT “”,
jswb decimal64(10, 0) NULL COMMENT “”,
kcsl decimal64(18, 6) NULL COMMENT “”,
materieldiscountrateid varchar(100) NULL COMMENT “”,
nocommission decimal64(1, 0) NULL COMMENT “”,
originalno varchar(100) NULL COMMENT “”,
originalnumber varchar(100) NULL COMMENT “”,
orignaltotal decimal64(18, 2) NULL COMMENT “”,
paydate datetime NULL COMMENT “”,
pcqj decimal64(10, 0) NULL COMMENT “”,
pfj decimal64(18, 6) NULL COMMENT “”,
pfml1 decimal64(18, 6) NULL COMMENT “”,
ph1_sl decimal64(18, 6) NULL COMMENT “”,
ph2_sl decimal64(18, 6) NULL COMMENT “”,
ph2_xq datetime NULL COMMENT “”,
ph3 varchar(120) NULL COMMENT “”,
ph3_xq datetime NULL COMMENT “”,
pm varchar(320) NULL COMMENT “”,
pm_zjm varchar(120) NULL COMMENT “”,
shry varchar(120) NULL COMMENT “”,
skje decimal64(18, 2) NULL COMMENT “”,
srsl decimal64(18, 6) NULL COMMENT “”,
syzblastpurchaseprice decimal64(18, 6) NULL COMMENT “”,
tele varchar(400) NULL COMMENT “”,
trandate datetime NULL COMMENT “”,
trandate_crm datetime NULL COMMENT “”,
tsrq date NULL COMMENT “”,
tzje decimal64(18, 2) NULL COMMENT “”,
tzjs decimal64(18, 2) NULL COMMENT “”,
tzse decimal64(18, 2) NULL COMMENT “”,
tzsl decimal64(18, 6) NULL COMMENT “”,
webjsfs decimal64(18, 2) NULL COMMENT “”,
webjsfs1 varchar(200) NULL COMMENT “”,
wsje decimal64(18, 2) NULL COMMENT “”,
wslsze decimal64(18, 2) NULL COMMENT “”,
xsr1 varchar(200) NULL COMMENT “”,
ye decimal64(18, 2) NULL COMMENT “”,
ysje decimal64(18, 2) NULL COMMENT “”,
zjm varchar(120) NULL COMMENT “”,
zrecphone varchar(100) NULL COMMENT “”,
ztflsx varchar(100) NULL COMMENT “”,
zzaddress varchar(500) NULL COMMENT “”,
zzcity varchar(100) NULL COMMENT “”,
zzdistrict varchar(100) NULL COMMENT “”,
zzrecname varchar(100) NULL COMMENT “”,
zzregion varchar(100) NULL COMMENT “”,
ftje decimal64(18, 6) NULL COMMENT “”,
istran decimal64(1, 0) NULL COMMENT “”,
amoeba_ml decimal64(18, 2) NULL COMMENT “”,
amoeba_sj decimal64(18, 2) NULL COMMENT “”,
scdw varchar(400) NULL COMMENT “”,
yyrcode varchar(100) NULL COMMENT “”,
yyrname varchar(100) NULL COMMENT “”,
relationship varchar(100) NULL COMMENT “”,
temperature varchar(100) NULL COMMENT “”,
amoeba_ml_his decimal64(18, 2) NULL COMMENT “”,
amoeba_ml_fhbz decimal64(1, 0) NULL COMMENT “”,
sfym varchar(20) NULL COMMENT “”,
c_test1 varchar(400) NULL COMMENT “”,
mbdj decimal64(10, 0) NULL COMMENT “”,
yscode varchar(80) NULL COMMENT “”,
department varchar(200) NULL COMMENT “”,
hospital varchar(200) NULL COMMENT “”,
physician varchar(200) NULL COMMENT “”,
posname varchar(100) NULL COMMENT “”,
patient varchar(200) NULL COMMENT “”,
disease varchar(200) NULL COMMENT “”,
trandate_dtp datetime NULL COMMENT “”,
online_type varchar(60) NULL COMMENT “”,
province varchar(400) NULL COMMENT “”,
city varchar(400) NULL COMMENT “”,
district varchar(400) NULL COMMENT “”,
street varchar(400) NULL COMMENT “”,
generation_date datetime NULL COMMENT “”,
sap_plant varchar(30) NULL COMMENT “”,
sap_barch varchar(60) NULL COMMENT “”,
op_ts datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT “数据同步时间”
) ENGINE=OLAP
PRIMARY KEY(subbh, dh, kdrq)
COMMENT “OLAP”
DISTRIBUTED BY HASH(kdrq) BUCKETS 24
PROPERTIES (
“replication_num” = “3”,
“in_memory” = “false”,
“storage_format” = “DEFAULT”,
“enable_persistent_index” = “false”,
“replicated_storage” = “true”,
“compression” = “LZ4”
);
【load 语句】
LOAD LABEL ods_g3_dbo_subfhd
(
DATA INFILE(“hdfs://sypmnameservice/user/hive/warehouse/syhc_dwd_global.db/dwd_g3_dbo_subfhd_inc_1d/dt=20230603/*”)
INTO TABLE ods_g3_dbo_subfhd_bak
COLUMNS TERMINATED BY “\u0001”
FORMAT as “orc”
(
subbh,
dh,
kdrq,
fid,
fyear,
djlx,
fperiod,
ftype,
jzbz,
shbz,
lsh,
lb,
hh,
zzsl,
sl,
zk,
dj,
lsj,
sj,
je,
se,
jshj,
cbj,
cbe,
ml,
lsze,
cj,
xsr,
bz,
czry,
ph1,
ph2,
ph3_sl,
ph1_xq,
fvchnum,
fgroup,
fvch,
stockid,
bm,
yhkh,
pcdh,
fpzdate,
fpzperiod,
fpzyear,
vchoperation,
fvchid,
jzry,
commissionrate,
scrq,
batchprice,
billtime,
jzrq,
balancebaseprice,
integralvalue,
posguid,
dealguid,
daliylogdate,
customerinfo,
zblastpurchaseprice,
zbjhdj,
headbatchnumber,
suppliercode,
cxfaid,
pfml,
hsml,
rightoptcode,
workshiftid,
tcfs,
cxfamxid,
kh,
policytype,
fsfcj,
cdbz,
personname,
idcardno,
boxid,
checknum,
isrecheck,
recheckdate,
recheckoperatorguid,
tjbh,
auditingname,
patientname,
doctorname,
doctorcode,
clname,
pharmacistname,
appdetailguid,
appguid,
address,
bh,
branchguid,
bs,
cd,
clfcr,
clfhr,
c_ftbz,
c_ftje,
c_ftjs,
c_ftml,
c_ftse,
c_zsfy,
dw,
dwmc,
dydh,
falreadyconsignnum,
fconsignbillmark,
fhbz,
fhrq,
fhsl,
fhsysl,
fk_tag,
fmanyconsignmark,
fp_tag,
fremainmcnum,
fwithdrawgoodsbillid,
fwithdrawgoodsmark,
gcbh,
gg,
gspxsy,
hksl,
hkwb,
hk_date,
hk_tag,
htbh,
hyj,
isauditing,
isqq,
istj,
istran_crm,
jj,
js,
jsfs,
jsje,
jsjs,
jsse,
jssl,
jswb,
kcsl,
materieldiscountrateid,
nocommission,
originalno,
originalnumber,
orignaltotal,
paydate,
pcqj,
pfj,
pfml1,
ph1_sl,
ph2_sl,
ph2_xq,
ph3,
ph3_xq,
pm,
pm_zjm,
shry,
skje,
srsl,
syzblastpurchaseprice,
tele,
trandate,
trandate_crm,
tsrq,
tzje,
tzjs,
tzse,
tzsl,
webjsfs,
webjsfs1,
wsje,
wslsze,
xsr1,
ye,
ysje,
zjm,
zrecphone,
ztflsx,
zzaddress,
zzcity,
zzdistrict,
zzrecname,
zzregion,
ftje,
istran,
amoeba_ml,
amoeba_sj,
scdw,
yyrcode,
yyrname,
relationship,
temperature,
amoeba_ml_his,
amoeba_ml_fhbz,
sfym,
c_test1,
mbdj,
yscode,
department,
hospital,
physician,
posname,
patient,
disease,
trandate_dtp,
online_type,
province,
city,
district,
street
)
SET
(
subbh=subbh,
dh=dh,
kdrq=kdrq,
fid=fid,
fyear=fyear,
djlx=djlx,
fperiod=fperiod,
ftype=ftype,
jzbz=jzbz,
shbz=shbz,
lsh=lsh,
lb=lb,
hh=hh,
zzsl=zzsl,
sl=sl,
zk=zk,
dj=dj,
lsj=lsj,
sj=sj,
je=je,
se=se,
jshj=jshj,
cbj=cbj,
cbe=cbe,
ml=ml,
lsze=lsze,
cj=cj,
xsr=xsr,
bz=bz,
czry=czry,
ph1=ph1,
ph2=ph2,
ph3_sl=ph3_sl,
ph1_xq=ph1_xq,
fvchnum=fvchnum,
fgroup=fgroup,
fvch=fvch,
stockid=stockid,
bm=bm,
yhkh=yhkh,
pcdh=pcdh,
fpzdate=fpzdate,
fpzperiod=fpzperiod,
fpzyear=fpzyear,
vchoperation=vchoperation,
fvchid=fvchid,
jzry=jzry,
commissionrate=commissionrate,
scrq=scrq,
batchprice=batchprice,
billtime=billtime,
jzrq=jzrq,
balancebaseprice=balancebaseprice,
integralvalue=integralvalue,
posguid=posguid,
dealguid=dealguid,
daliylogdate=daliylogdate,
customerinfo=customerinfo,
zblastpurchaseprice=zblastpurchaseprice,
zbjhdj=zbjhdj,
headbatchnumber=headbatchnumber,
suppliercode=suppliercode,
cxfaid=cxfaid,
pfml=pfml,
hsml=hsml,
rightoptcode=rightoptcode,
workshiftid=workshiftid,
tcfs=tcfs,
cxfamxid=cxfamxid,
kh=kh,
policytype=policytype,
fsfcj=fsfcj,
cdbz=cdbz,
personname=personname,
idcardno=idcardno,
boxid=boxid,
checknum=checknum,
isrecheck=isrecheck,
recheckdate=recheckdate,
recheckoperatorguid=recheckoperatorguid,
tjbh=tjbh,
auditingname=auditingname,
patientname=patientname,
doctorname=doctorname,
doctorcode=doctorcode,
clname=clname,
pharmacistname=pharmacistname,
appdetailguid=appdetailguid,
appguid=appguid,
address=address,
bh=bh,
branchguid=branchguid,
bs=bs,
cd=cd,
clfcr=clfcr,
clfhr=clfhr,
c_ftbz=c_ftbz,
c_ftje=c_ftje,
c_ftjs=c_ftjs,
c_ftml=c_ftml,
c_ftse=c_ftse,
c_zsfy=c_zsfy,
dw=dw,
dwmc=dwmc,
dydh=dydh,
falreadyconsignnum=falreadyconsignnum,
fconsignbillmark=fconsignbillmark,
fhbz=fhbz,
fhrq=fhrq,
fhsl=fhsl,
fhsysl=fhsysl,
fk_tag=fk_tag,
fmanyconsignmark=fmanyconsignmark,
fp_tag=fp_tag,
fremainmcnum=fremainmcnum,
fwithdrawgoodsbillid=fwithdrawgoodsbillid,
fwithdrawgoodsmark=fwithdrawgoodsmark,
gcbh=gcbh,
gg=gg,
gspxsy=gspxsy,
hksl=hksl,
hkwb=hkwb,
hk_date=hk_date,
hk_tag=hk_tag,
htbh=htbh,
hyj=hyj,
isauditing=isauditing,
isqq=isqq,
istj=istj,
istran_crm=istran_crm,
jj=jj,
js=js,
jsfs=jsfs,
jsje=jsje,
jsjs=jsjs,
jsse=jsse,
jssl=jssl,
jswb=jswb,
kcsl=kcsl,
materieldiscountrateid=materieldiscountrateid,
nocommission=nocommission,
originalno=originalno,
originalnumber=originalnumber,
orignaltotal=orignaltotal,
paydate=paydate,
pcqj=pcqj,
pfj=pfj,
pfml1=pfml1,
ph1_sl=ph1_sl,
ph2_sl=ph2_sl,
ph2_xq=ph2_xq,
ph3=ph3,
ph3_xq=ph3_xq,
pm=pm,
pm_zjm=pm_zjm,
shry=shry,
skje=skje,
srsl=srsl,
syzblastpurchaseprice=syzblastpurchaseprice,
tele=tele,
trandate=trandate,
trandate_crm=trandate_crm,
tsrq=tsrq,
tzje=tzje,
tzjs=tzjs,
tzse=tzse,
tzsl=tzsl,
webjsfs=webjsfs,
webjsfs1=webjsfs1,
wsje=wsje,
wslsze=wslsze,
xsr1=xsr1,
ye=ye,
ysje=ysje,
zjm=zjm,
zrecphone=zrecphone,
ztflsx=ztflsx,
zzaddress=zzaddress,
zzcity=zzcity,
zzdistrict=zzdistrict,
zzrecname=zzrecname,
zzregion=zzregion,
ftje=ftje,
istran=istran,
amoeba_ml=amoeba_ml,
amoeba_sj=amoeba_sj,
scdw=scdw,
yyrcode=yyrcode,
yyrname=yyrname,
relationship=relationship,
temperature=temperature,
amoeba_ml_his=amoeba_ml_his,
amoeba_ml_fhbz=amoeba_ml_fhbz,
sfym=sfym,
c_test1=c_test1,
mbdj=mbdj,
yscode=yscode,
department=department,
hospital=hospital,
physician=physician,
posname=posname,
patient=patient,
disease=disease,
trandate_dtp=trandate_dtp,
online_type=online_type,
province=province,
city=city,
district=district,
street=street
)
)
WITH BROKER
(
“username” = “root”,
“password” = “”
)
PROPERTIES
(
“timeout”=“10800”
);
【StarRocks版本】3.0.1 a1f411b
【集群规模】例如：1fe+3be（分开部署）
【机器信息】be 32核256g内存 fe8核50g内存
【联系方式】微信号：tsqs777

[jingdan]

这个问题可以试试admin set frontend config (“enable_pipeline_load”=“false”)，如果可以的话，3.0.2会修复