【详述】从SR2.0.0-GA通过HDFS EXPORT TABLE方式导出，然后通过LOAD LABEL导入到SR3.1.7报错：type:LOAD_RUN_FAIL; msg:CSV line length exceed limit 536870912
【背景】SR2.0.0-GA集群迁移到SR3.2.3集群
【业务影响】
【是否存算分离】否
【StarRocks版本】SR2.0.0-GA SR3.2.3
【connector版本】
【集群规模】3fe（1 leader +2follower）+3be（fe与be混部）
【机器信息】32C/100G/万兆
【联系方式】社区群3-小蚂蚁
【附件】
- 导入报错
1、导入进度非100%，报错：type:LOAD_RUN_FAIL; msg:CSV line length exceed limit 536870912
2、be.INFO.log日志如下：
I0219 17:34:51.754664 3050 pipeline_driver_executor.cpp:327] [Driver] Succeed to report exec state: fragment_instance_id=08d480dc-fdcd-4618-87fd-c220301d1433
E0219 17:34:55.097247 3100 scan_operator.cpp:412] scan fragment 08d480dc-fdcd-4618-87fd-c220301d1433 driver 4 Scan tasks error: Internal error: CSV line length exceed limit 536870912
/build/starrocks/be/src/formats/csv/csv_reader.cpp:501 _expand_buffer()
W0219 17:34:55.097559 3081 pipeline_driver.cpp:308] pull_chunk returns not ok status Internal error: CSV line length exceed limit 536870912
/build/starrocks/be/src/formats/csv/csv_reader.cpp:501 _expand_buffer()
/build/starrocks/be/src/exec/pipeline/scan/scan_operator.cpp:237 _get_scan_status()
W0219 17:34:55.097577 3081 pipeline_driver_executor.cpp:162] [Driver] Process error, query_id=08d480dc-fdcd-4618-87fd-c220301d1430, instance_id=08d480dc-fdcd-4618-87fd-c220301d1433, status=Internal error: CSV line length exceed limit 536870912
/build/starrocks/be/src/formats/csv/csv_reader.cpp:501 _expand_buffer()
/build/starrocks/be/src/exec/pipeline/scan/scan_operator.cpp:237 _get_scan_status()
I0219 17:34:55.097600 3081 profile_report_worker.cpp:60] unregister_pipeline_load query_id=08d480dc-fdcd-4618-87fd-c220301d1430, fragment_instance_id=08d480dc-fdcd-4618-87fd-c220301d1433
W0219 17:34:55.097841 3052 tablet_sink.cpp:1686] close channel failed. channel_name=NodeChannel[10005], load_info=load_id=08d480dc-fdcd-4618-87fd-c220301d1430, txn_id: 3007, parallel=1, compress_type=2, error_msg=Cancelled by pipeline engine
建表语句如下：
CREATE TABLE demo_test (
start_date date NOT NULL COMMENT “",
affiance_plan_no varchar(60) NOT NULL COMMENT "A编号",
loan_id varchar(64) NOT NULL COMMENT "据号",
cert_no varchar(100) NOT NULL COMMENT “码”,
loan_type varchar(2) NOT NULL COMMENT “类型”,
balance_transfer varchar(1) NOT NULL COMMENT “标识”,
name varchar(100) NOT NULL COMMENT "客",
cur_date varchar(8) NULL COMMENT “账日期”,
fx_account_id varchar(64) NULL COMMENT “x”,
trade_no varchar(64) NULL COMMENT “xx流号”,
end_date varchar(8) NULL COMMENT "到”,
total_terms int(11) NULL COMMENT “总*”,
encash_amt decimal64(16, 2) NULL COMMENT “额",
repay_mode varchar(1) NULL COMMENT "方”,
usage varchar(2) NULL COMMENT “2”,
int_rate varchar(10) NULL COMMENT “23",
ovd_rate varchar(6) NULL COMMENT "利”,
capital_set_no varchar(64) NULL COMMENT “编码”,
loan_time varchar(32) NULL COMMENT “XX时*”,
commodity varchar(256) NULL COMMENT “商息",
counterparty_name varchar(256) NULL COMMENT “xx名”,
counterparty_bank varchar(8) NULL COMMENT “xx机”,
counterparty_account varchar(64) NULL COMMENT “xx账号”,
asset_side varchar(10) NULL COMMENT “",
capital_side varchar(10) NULL COMMENT "合”,
prd_code varchar(16) NULL COMMENT “2”,
openday varchar(10) NULL COMMENT "业”,
input_time varchar(20) NULL COMMENT “创建*”,
cert_type varchar(4) NULL COMMENT “类型",
open_date varchar(10) NULL COMMENT "时”,
bill_day varchar(2) NULL COMMENT “XX日”,
due_day varchar(2) NULL COMMENT “XX日2”,
affiance_plan_term_num varchar(8) NULL COMMENT “XX号”
) ENGINE=OLAP
UNIQUE KEY(start_date, affiance_plan_no, loan_id, cert_no, loan_type, balance_transfer, name)
COMMENT “XX表-XX”
PARTITION BY RANGE(start_date)
(PARTITION p202211 VALUES [(‘2022-11-01’), (‘2022-12-01’)),
PARTITION p202212 VALUES [(‘2022-12-01’), (‘2023-01-01’)),
PARTITION p202301 VALUES [(‘2023-01-01’), (‘2023-02-01’)),
PARTITION p202302 VALUES [(‘2023-02-01’), (‘2023-03-01’)),
PARTITION p202303 VALUES [(‘2023-03-01’), (‘2023-04-01’)),
PARTITION p202304 VALUES [(‘2023-04-01’), (‘2023-05-01’)),
PARTITION p202305 VALUES [(‘2023-05-01’), (‘2023-06-01’)),
PARTITION p202306 VALUES [(‘2023-06-01’), (‘2023-07-01’)),
PARTITION p202307 VALUES [(‘2023-07-01’), (‘2023-08-01’)),
PARTITION p202308 VALUES [(‘2023-08-01’), (‘2023-09-01’)),
PARTITION p202309 VALUES [(‘2023-09-01’), (‘2023-10-01’)),
PARTITION p202310 VALUES [(‘2023-10-01’), (‘2023-11-01’)),
PARTITION p202311 VALUES [(‘2023-11-01’), (‘2023-12-01’)),
PARTITION p202312 VALUES [(‘2023-12-01’), (‘2024-01-01’)),
PARTITION p202401 VALUES [(‘2024-01-01’), (‘2024-02-01’)),
PARTITION p202402 VALUES [(‘2024-02-01’), (‘2024-03-01’)),
PARTITION p202403 VALUES [(‘2024-03-01’), (‘2024-04-01’)),
PARTITION p202404 VALUES [(‘2024-04-01’), (‘2024-05-01’)),
PARTITION p202405 VALUES [(‘2024-05-01’), (‘2024-06-01’)),
PARTITION p202406 VALUES [(‘2024-06-01’), (‘2024-07-01’)),
PARTITION p202407 VALUES [(‘2024-07-01’), (‘2024-08-01’)),
PARTITION p202408 VALUES [(‘2024-08-01’), (‘2024-09-01’)),
PARTITION p202409 VALUES [(‘2024-09-01’), (‘2024-10-01’)),
PARTITION p202410 VALUES [(‘2024-10-01’), (‘2024-11-01’)),
PARTITION p202411 VALUES [(‘2024-11-01’), (‘2024-12-01’)),
PARTITION p202412 VALUES [(‘2024-12-01’), (‘2025-01-01’)))
DISTRIBUTED BY HASH(loan_id) BUCKETS 24
PROPERTIES (
“replication_num” = “3”,
“bloom_filter_columns” = “balance_transfer, loan_type”,
“dynamic_partition.enable” = “true”,
“dynamic_partition.time_unit” = “MONTH”,
“dynamic_partition.time_zone” = “Asia/Shanghai”,
“dynamic_partition.start” = “-2147483648”,
“dynamic_partition.end” = “3”,
“dynamic_partition.prefix” = “p”,
“dynamic_partition.buckets” = “24”,
“dynamic_partition.start_day_of_month” = “1”,
“in_memory” = “false”,
“storage_format” = “DEFAULT”
);
随机抽取一条语句：
INSERT INTO demo_test (
start_date,
affiance_plan_no,
loan_id,
cert_no,
loan_type,
balance_transfer,
name,
cur_date,
fx_account_id,
trade_no,
end_date,
total_terms,
encash_amt,
repay_mode,
usage,
int_rate,
ovd_rate,
capital_set_no,
loan_time,
commodity,
counterparty_name,
counterparty_bank,
counterparty_account,
asset_side,
capital_side,
prd_code,
openday,
input_time,
cert_type,
open_date,
bill_day,
due_day,
affiance_plan_term_num
)
VALUES
(
‘2024-01-22’,
‘ABCCTC2022X3499’,
‘KO202401222241279920008288222’,
‘211004199401103322’,
‘1’,
‘N’,
‘张三’,
‘20240122’,
‘1000272849056305457608288212’,
‘TP2024012222410401071185945555’,
‘20240208’,
1,
127.95,
‘3’,
‘1’,
‘0’,
‘0.05’,
‘Z209003’,
‘2024-01-22 22:41:28’,
‘永远的神\n’,
‘哈喽某X*X’,
NULL,
‘6020230413435275’,
‘N060001010’,
‘M060000001’,
‘YYDS-DGYF005’,
‘2024-01-22’,
‘2024-01-23 07:56:23’,
‘1’,
‘20231021’,
‘1’,
‘6’,
‘1’
);