3.3.6版本 iceberg写入数据时内存占用非常高, 已经使用了1并行度, 内存还是打满了, arrow内存占用特别高

colagy · 2024年11月27日 13:55

执行insert into target select * from source; 从内表写入到iceberg只设置了并行度1
看上去主要被arrow占用了

内存dump


digraph "/opt/starrocks/be/lib/starrocks_be; 67617.0 MB" {                                                                                                                                                
 node [width=0.375,height=0.25];                                                                                                                                                                           
 Legend [shape=box,fontsize=24,shape=plaintext,label="/opt/starrocks/be/lib/starrocks_be\lTotal MB: 67617.0\lFocusing on: 67617.0\lDropped nodes with <= 338.1 abs(MB)\lDropped edges with <= 67.6 MB\l"]; 
 N1 [label="__clone@@GLIBC_2.2.5\n0.0 (0.0%)\rof 67616.5 (100.0%)\r",shape=box,fontsize=8.0];                                                                                                              
 N2 [label="pthread_condattr_setpshared@GLIBC_2.2.5\n0.0 (0.0%)\rof 67616.5 (100.0%)\r",shape=box,fontsize=8.0];                                                                                           
 N3 [label="starrocks\nThread\nsupervise_thread\n0.0 (0.0%)\rof 67561.3 (99.9%)\r",shape=box,fontsize=8.0];                                                                                                
 N4 [label="starrocks\nThreadPool\ndispatch_thread\n0.0 (0.0%)\rof 67561.3 (99.9%)\r",shape=box,fontsize=8.0];                                                                                             
 N5 [label="starrocks\npipeline\nGlobalDriverExecutor\n_worker_thread\n0.0 (0.0%)\rof 66121.8 (97.8%)\r",shape=box,fontsize=8.0];                                                                          
 N6 [label="starrocks\npipeline\nPipelineDriver\nprocess\n0.0 (0.0%)\rof 66121.8 (97.8%)\r",shape=box,fontsize=8.0];                                                                                       
 N7 [label="starrocks\nconnector\nConnectorChunkSink\nadd\n0.0 (0.0%)\rof 66038.5 (97.7%)\r",shape=box,fontsize=8.0];                                                                                      
 N8 [label="starrocks\npipeline\nConnectorSinkOperator\npush_chunk\n0.0 (0.0%)\rof 66038.5 (97.7%)\r",shape=box,fontsize=8.0];                                                                             
 N9 [label="starrocks\nformats\nParquetFileWriter\nwrite\n0.0 (0.0%)\rof 65702.2 (97.2%)\r",shape=box,fontsize=8.0];                                                                                       
 N10 [label="posix_memalign\n57376.5 (84.9%)\r",shape=box,fontsize=54.1];                                                                                                                                  
 N11 [label="starrocks\nArrowMemoryPool\nAllocate\n0.0 (0.0%)\rof 57376.5 (84.9%)\r",shape=box,fontsize=8.0];                                                                                              
 N12 [label="starrocks\nparquet\nChunkWriter\nwrite\n0.0 (0.0%)\rof 56337.9 (83.3%)\r",shape=box,fontsize=8.0];                                                                                            
 N13 [label="starrocks\nparquet\nLevelBuilder\n_write_column_chunk\n0.0 (0.0%)\rof 56337.9 (83.3%)\r",shape=box,fontsize=8.0];                                                                             
 N14 [label="starrocks\nparquet\nLevelBuilder\nwrite\n0.0 (0.0%)\rof 56337.9 (83.3%)\r",shape=box,fontsize=8.0];                                                                                           
 N15 [label="std\n_Function_handler\n_M_invoke@78761b0\n0.0 (0.0%)\rof 56337.9 (83.3%)\r",shape=box,fontsize=8.0];                                                                                         
 N16 [label="arrow\nPoolBuffer\nReserve\n0.0 (0.0%)\rof 56092.4 (83.0%)\r",shape=box,fontsize=8.0];                                                                                                        
 N17 [label="arrow\nPoolBuffer\nResize\n0.0 (0.0%)\rof 56092.4 (83.0%)\r",shape=box,fontsize=8.0];                                                                                                         
 N18 [label="parquet\nTypedColumnWriterImpl\nWriteBatch\n0.0 (0.0%)\rof 55787.3 (82.5%)\r",shape=box,fontsize=8.0];                                                                                        
 N19 [label="parquet\nTypedColumnWriterImpl\nWriteBatch\n{lambda#1}\noperator\n0.0 (0.0%)\rof 55787.3 (82.5%)\r",shape=box,fontsize=8.0];                                                                  
 N20 [label="starrocks\nArrowMemoryPool\nReallocate\n0.0 (0.0%)\rof 51795.2 (76.6%)\r",shape=box,fontsize=8.0];                                                                                            
 N21 [label="parquet\nColumnWriterImpl\nAddDataPage\n[clone\n.localalias]\n0.0 (0.0%)\rof 43615.1 (64.5%)\r",shape=box,fontsize=8.0];                                                                      
 N22 [label="parquet\nColumnWriterImpl\nBuildDataPageV1\n[clone\n.localalias]\n0.0 (0.0%)\rof 43614.1 (64.5%)\r",shape=box,fontsize=8.0];                                                                  
 N23 [label="arrow\nio\nBufferOutputStream\nReserve\n[clone\n.localalias]\n0.0 (0.0%)\rof 39793.5 (58.9%)\r",shape=box,fontsize=8.0];                                                                      
 N24 [label="arrow\nio\nBufferOutputStream\nWrite\n[clone\n.localalias]\n0.0 (0.0%)\rof 39793.5 (58.9%)\r",shape=box,fontsize=8.0];                                                                        
 N25 [label="parquet\nSerializedPageWriter\nWriteDataPage\n0.0 (0.0%)\rof 33678.5 (49.8%)\r",shape=box,fontsize=8.0];                                                                                      
 N26 [label="parquet\nColumnWriterImpl\nClose\n[clone\n.localalias]\n0.0 (0.0%)\rof 7826.0 (11.6%)\r",shape=box,fontsize=8.0];                                                                             
 N27 [label="parquet\nRowGroupWriter\nClose\n0.0 (0.0%)\rof 7826.0 (11.6%)\r",shape=box,fontsize=8.0];                                                                                                     
 N28 [label="starrocks\nformats\nParquetFileWriter\n_flush_row_group\n0.0 (0.0%)\rof 7826.0 (11.6%)\r",shape=box,fontsize=8.0];                                                                            
 N29 [label="parquet\nBufferedPageWriter\nClose\n0.0 (0.0%)\rof 7808.0 (11.5%)\r",shape=box,fontsize=8.0];                                                                                                 
 N30 [label="starrocks\nio\nAsyncFlushOutputStream\nwrite\n0.0 (0.0%)\rof 7803.0 (11.5%)\r",shape=box,fontsize=8.0];                                                                                       
 N31 [label="starrocks\nparquet\nAsyncParquetOutputStream\nWrite@786e860\n0.0 (0.0%)\rof 7803.0 (11.5%)\r",shape=box,fontsize=8.0];                                                                        
 N32 [label="starrocks\nparquet\nAsyncParquetOutputStream\nWrite@786eee0\n0.0 (0.0%)\rof 7803.0 (11.5%)\r",shape=box,fontsize=8.0];                                                                        
 N33 [label="std\nvector\n_M_default_append@53ed810\n7803.0 (11.5%)\r",shape=box,fontsize=25.0];                                                                                                           
 N34 [label="parquet\nSerializedPageWriter\nWriteDictionaryPage\n0.0 (0.0%)\rof 6137.5 (9.1%)\r",shape=box,fontsize=8.0];                                                                                  
 N35 [label="parquet\nTypedColumnWriterImpl\nWriteDictionaryPage\n0.0 (0.0%)\rof 6137.5 (9.1%)\r",shape=box,fontsize=8.0];                                                                                 
 N36 [label="parquet\n\nDictEncoderImpl\nPutByteArray\n0.0 (0.0%)\rof 5671.6 (8.4%)\r",shape=box,fontsize=8.0];                                                                                            
 N37 [label="arrow\nBufferBuilder\nResize\n6.5 (0.0%)\rof 5520.3 (8.2%)\r",shape=box,fontsize=8.5];                                                                                                        
 N38 [label="parquet\n\nDictEncoderImpl\nPut\n0.0 (0.0%)\rof 5254.4 (7.8%)\r",shape=box,fontsize=8.0];                                                                                                     
 N39 [label="arrow\nAllocateResizableBuffer\n[clone\n.localalias]\n0.0 (0.0%)\rof 4417.7 (6.5%)\r",shape=box,fontsize=8.0];                                                                                
 N40 [label="arrow\ninternal\nHashTable\nUpsize\n0.0 (0.0%)\rof 3192.4 (4.7%)\r",shape=box,fontsize=8.0];                                                                                                  
 N41 [label="parquet\nFileSerializer\nAppendRowGroup\n0.0 (0.0%)\rof 1532.3 (2.3%)\r",shape=box,fontsize=8.0];                                                                                             
 N42 [label="starrocks\npipeline\nChunkSource\nbuffer_next_batch_chunks_blocking\n0.0 (0.0%)\rof 1437.6 (2.1%)\r",shape=box,fontsize=8.0];                                                                 
 N43 [label="starrocks\npipeline\nOlapChunkSource\n_read_chunk\n0.0 (0.0%)\rof 1437.6 (2.1%)\r",shape=box,fontsize=8.0];                                                                                   
 N44 [label="starrocks\npipeline\nScanOperator\n_trigger_next_scan\n{lambda#1}\noperator\n[clone\n.constprop.0]\n0.0 (0.0%)\rof 1437.6 (2.1%)\r",shape=box,fontsize=8.0];                                  
 N45 [label="starrocks\nworkgroup\nScanExecutor\nworker_thread\n0.0 (0.0%)\rof 1437.6 (2.1%)\r",shape=box,fontsize=8.0];                                                                                   
 N46 [label="parquet\nRowGroupSerializer\nCreateColumnWriterForColumn\n0.0 (0.0%)\rof 1436.2 (2.1%)\r",shape=box,fontsize=8.0];                                                                            
 N47 [label="starrocks\nSegmentIterator\ndo_get_next\n0.0 (0.0%)\rof 1370.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                             
 N48 [label="starrocks\nSegmentIteratorWrapper\ndo_get_next\n0.0 (0.0%)\rof 1370.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                      
 N49 [label="starrocks\nTabletReader\ndo_get_next\n0.0 (0.0%)\rof 1370.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                                
 N50 [label="starrocks\nTimedChunkIterator\ndo_get_next\n0.0 (0.0%)\rof 1370.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                          
 N51 [label="starrocks\npipeline\nOlapChunkSource\n_read_chunk_from_storage\n0.0 (0.0%)\rof 1370.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                      
 N52 [label="starrocks\nParsedPageV2\nread\n0.0 (0.0%)\rof 1336.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                                       
 N53 [label="starrocks\nScalarColumnIterator\nnext_batch\n0.0 (0.0%)\rof 1336.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                                         
 N54 [label="starrocks\nSegmentIterator\n_do_get_next\n[clone\n.constprop.0]\n0.0 (0.0%)\rof 1336.5 (2.0%)\r",shape=box,fontsize=8.0];                                                                     
 N55 [label="std\nvector\n_M_realloc_insert@b1e4f00\n0.0 (0.0%)\rof 1284.0 (1.9%)\r",shape=box,fontsize=8.0];                                                                                              
 N56 [label="parquet\nColumnWriter\nMake\n26.0 (0.0%)\rof 1197.1 (1.8%)\r",shape=box,fontsize=9.0];                                                                                                        
 N57 [label="arrow\nBaseBinaryBuilder\nAppend\n0.0 (0.0%)\rof 1195.2 (1.8%)\r",shape=box,fontsize=8.0];                                                                                                    
 N58 [label="parquet\nTypedColumnWriterImpl\nTypedColumnWriterImpl@b24aa40\n0.0 (0.0%)\rof 1156.8 (1.7%)\r",shape=box,fontsize=8.0];                                                                       
 N59 [label="parquet\nMakeEncoder\n21.5 (0.0%)\rof 1053.6 (1.6%)\r",shape=box,fontsize=8.9];                                                                                                               
 N60 [label="arrow\ninternal\nBinaryMemoTable\nBinaryMemoTable\n0.0 (0.0%)\rof 1018.3 (1.5%)\r",shape=box,fontsize=8.0];                                                                                   
 N61 [label="std\nvector\n_M_default_append@54159b0\n986.9 (1.5%)\r",shape=box,fontsize=14.0];                                                                                                             
 N62 [label="starrocks\nBinaryPlainPageDecoder\nappend_range\n0.0 (0.0%)\rof 930.6 (1.4%)\r",shape=box,fontsize=8.0];                                                                                      
 N63 [label="starrocks\nBinaryPlainPageDecoder\nnext_batch@6649460\n0.0 (0.0%)\rof 930.6 (1.4%)\r",shape=box,fontsize=8.0];                                                                                
 N64 [label="parquet\nColumnWriterImpl\nWriteDefinitionLevels\n0.0 (0.0%)\rof 744.0 (1.1%)\r",shape=box,fontsize=8.0];                                                                                     
 N65 [label="starrocks\nparquet\nColumnChunkWriter\nwrite\n0.0 (0.0%)\rof 550.6 (0.8%)\r",shape=box,fontsize=8.0];                                                                                         
 N66 [label="parquet\nTypedColumnWriterImpl\nWriteBatchSpaced\n{lambda#1}\noperator@b24c000\n0.0 (0.0%)\rof 540.4 (0.8%)\r",shape=box,fontsize=8.0];                                                       
 N67 [label="parquet\nTypedColumnWriterImpl\nWriteBatchSpaced@b24c3b0\n0.0 (0.0%)\rof 540.4 (0.8%)\r",shape=box,fontsize=8.0];                                                                             
 N68 [label="parquet\n\nDictEncoderImpl\nPutSpaced\n0.0 (0.0%)\rof 417.2 (0.6%)\r",shape=box,fontsize=8.0];                                                                                                
 N69 [label="starrocks\nBinaryPlainPageDecoder\nnext_batch@664a040\n0.0 (0.0%)\rof 345.5 (0.5%)\r",shape=box,fontsize=8.0];                                                                                
 N70 [label="starrocks\nNullableColumn\nappend_continuous_strings\n0.0 (0.0%)\rof 345.5 (0.5%)\r",shape=box,fontsize=8.0];                                                                                 
 N71 [label="starrocks\nObjectColumn\nappend_strings\n0.0 (0.0%)\rof 345.5 (0.5%)\r",shape=box,fontsize=8.0];                                                                                              
 N72 [label="std\n__cxx11\nbasic_string\n_M_mutate\n342.1 (0.5%)\r",shape=box,fontsize=11.6];                                                                                                              
 N73 [label="std\n__cxx11\nbasic_string\n_M_replace\n0.0 (0.0%)\rof 342.1 (0.5%)\r",shape=box,fontsize=8.0];                                                                                               
 N1 -> N2 [label=67616.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N3 -> N4 [label=67561.3, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N2 -> N3 [label=67561.3, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N5 -> N6 [label=66121.8, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N4 -> N5 [label=66121.8, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N6 -> N8 [label=66038.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N8 -> N7 [label=66038.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N7 -> N9 [label=65702.2, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                  
 N11 -> N10 [label=57376.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N14 -> N13 [label=56337.9, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N9 -> N12 [label=56337.9, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                 
 N13 -> N15 [label=56337.9, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N12 -> N14 [label=56337.9, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N17 -> N16 [label=56092.4, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N18 -> N19 [label=55787.3, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N15 -> N18 [label=55787.3, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N20 -> N11 [label=51795.2, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N16 -> N20 [label=51795.2, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N21 -> N22 [label=43614.1, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N19 -> N21 [label=43593.9, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N24 -> N23 [label=39793.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N23 -> N17 [label=39793.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N25 -> N24 [label=33656.0, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N22 -> N25 [label=33626.5, weight=100000, style="setlinewidth(2.000000)"];                                                                                                                                
 N22 -> N17 [label=9964.8, weight=100000, style="setlinewidth(0.884230)"];                                                                                                                                 
 N9 -> N28 [label=7826.0, weight=100000, style="setlinewidth(0.694442)"];                                                                                                                                  
 N27 -> N26 [label=7826.0, weight=100000, style="setlinewidth(0.694442)"];                                                                                                                                 
 N28 -> N27 [label=7826.0, weight=100000, style="setlinewidth(0.694442)"];                                                                                                                                 
 N26 -> N29 [label=7808.0, weight=100000, style="setlinewidth(0.692845)"];                                                                                                                                 
 N30 -> N33 [label=7803.0, weight=100000, style="setlinewidth(0.692401)"];                                                                                                                                 
 N32 -> N31 [label=7803.0, weight=100000, style="setlinewidth(0.692401)"];                                                                                                                                 
 N31 -> N30 [label=7803.0, weight=100000];                                                                                                                                 
 N29 -> N32 [label=7803.0, weight=100000];                                                                                                                                 
 N19 -> N35 [label=6137.5, weight=100000];                                                                                                                                 
 N34 -> N24 [label=6137.5, weight=100000];                                                                                                                                 
 N35 -> N34 [label=6137.5, weight=100000];                                                                                                                                 
 N19 -> N38 [label=5254.4, weight=100000];                                                                                                                                 
 N38 -> N36 [label=5254.4, weight=100000];                                                                                                                                 
 N39 -> N17 [label=4297.2, weight=100000];                                                                                                                                 
 N16 -> N11 [label=4297.2, weight=100000];                                                                                                                                 
 N37 -> N39 [label=4218.5, weight=100000];                                                                                                                                 
 N36 -> N40 [label=3192.4, weight=100000];                                                                                                                                 
 N40 -> N37 [label=3192.4, weight=100000];                                                                                                                                 
 N9 -> N41 [label=1532.3, weight=100000];                                                                                                                                  
 N44 -> N42 [label=1437.6, weight=100000];                                                                                                                                 
 N4 -> N45 [label=1437.6, weight=100000];                                                                                                                                  
 N45 -> N44 [label=1437.6, weight=100000];                                                                                                                                 
 N42 -> N43 [label=1437.6, weight=100000];                                                                                                                                 
 N41 -> N46 [label=1436.2, weight=100000];                                                                                                                                 
 N50 -> N48 [label=1370.5, weight=100000];                                                                                                                                 
 N51 -> N49 [label=1370.5, weight=100000];                                                                                                                                 
 N48 -> N47 [label=1370.5, weight=100000];                                                                                                                                 
 N43 -> N51 [label=1370.5, weight=100000];                                                                                                                                 
 N49 -> N50 [label=1370.0, weight=100000];                                                                                                                                 
 N53 -> N52 [label=1336.5, weight=100000];                                                                                                                                 
 N47 -> N54 [label=1336.5, weight=100000];                                                                                                                                 
 N54 -> N53 [label=1336.5, weight=100000];                                                                                                                                 
 N37 -> N17 [label=1295.3, weight=100000];                                                                                                                                 
 N55 -> N11 [label=1284.0, weight=100000];                                                                                                                                 
 N36 -> N55 [label=1284.0, weight=100000];                                                                                                                                 
 N46 -> N56 [label=1197.1, weight=100000];                                                                                                                                 
 N36 -> N57 [label=1195.2, weight=100000];                                                                                                                                 
 N56 -> N58 [label=1156.8, weight=100000];                                                                                                                                 
 N58 -> N59 [label=1039.8, weight=100000];                                                                                                                                 
 N59 -> N60 [label=1018.3, weight=100000];                                                                                                                                 
 N60 -> N37 [label=1005.3, weight=100000];                                                                                                                                 
 N62 -> N61 [label=930.6, weight=100000];                                                                                                                                  
 N63 -> N62 [label=930.6, weight=100000];                                                                                                                                  
 N52 -> N63 [label=930.6, weight=100000];                                                                                                                                  
 N57 -> N37 [label=904.6, weight=100000];                                                                                                                                  
 N64 -> N17 [label=687.8, weight=100000];                                                                                                                                  
 N19 -> N64 [label=610.6, weight=100000];                                                                                                                                  
 N15 -> N65 [label=550.6, weight=100000];                                                                                                                                  
 N67 -> N66 [label=540.4, weight=100000];                                                                                                                                  
 N65 -> N67 [label=540.4, weight=100000];                                                                                                                                  
 N68 -> N36 [label=417.2, weight=100000];                                                                                                                                  
 N66 -> N68 [label=417.2, weight=100000];                                                                                                                                  
 N52 -> N69 [label=345.5, weight=100000];                                                                                                                                  
 N69 -> N70 [label=345.5, weight=100000];                                                                                                                                  
 N70 -> N71 [label=345.5, weight=100000];                                                                                                                                  
 N73 -> N72 [label=342.1, weight=100000];                                                                                                                                  
 N71 -> N73 [label=342.1, weight=100000];                                                                                                                                  
 N66 -> N64 [label=123.2, weight=100000];                                                                                                                                  
 }

zombee0 · 2024年11月28日 02:41

麻烦发一下其他信息，方便我们进行问题分析：

机器规格
集群配置有哪些修改
source 表和target表的schema信息，数据量
query涉及的分区数

colagy · 2024年11月28日 06:24

机器规格: fe:8c22G1 be:16C128G3
集群配置有哪些修改
Variables: parallel_fragment_exec_instance_num=1,max_parallel_scan_instance_num=-1,pipeline_dop=1,enable_adaptive_sink_dop=true,enable_runtime_adaptive_dop=false,runtime_profile_report_interval=10,resource_group=default_wg
- NonDefaultSessionVariables: {“use_page_cache”:{“defaultValue”:true,“actualValue”:false},“enable_view_based_mv_rewrite”:{“defaultValue”:true,“actualValue”:false},“big_query_profile_threshold”:{“defaultValue”:“0s”,“actualValue”:“1s”},“enable_scan_datacache”:{“defaultValue”:true,“actualValue”:false},“query_timeout”:{“defaultValue”:300,“actualValue”:14400},“catalog”:{“defaultValue”:“default_catalog”,“actualValue”:“t1111_iceberg”},“new_planner_optimize_timeout”:{“defaultValue”:3000,“actualValue”:10000},“enable_adaptive_sink_dop”:{“defaultValue”:false,“actualValue”:true},“cbo_decimal_cast_string_strict”:{“defaultValue”:true,“actualValue”:false},“enable_spill”:{“defaultValue”:false,“actualValue”:true},“consistent_hash_virtual_number”:{“defaultValue”:256,“actualValue”:128},“enable_datacache_async_populate_mode”:{“defaultValue”:true,“actualValue”:false},“pipeline_dop”:{“defaultValue”:0,“actualValue”:4},“enable_profile”:{“defaultValue”:false,“actualValue”:true},“enable_datacache_io_adaptor”:{“defaultValue”:true,“actualValue”:false},“cbo_eq_base_type”:{“defaultValue”:“decimal”,“actualValue”:“varchar”},“enable_file_metacache”:{“defaultValue”:true,“actualValue”:false},“group_concat_max_len”:{“defaultValue”:1024,“actualValue”:10000}}
source 表和target表的schema信息，数据量
数据量: 6000万
source schema:

CREATE TABLE `ods_api_wdtqjqm_sale_order_info_f` (
  `dt` datetime NOT NULL COMMENT "日分区",
  `__id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT "__id",
  `trade_id` varchar(1048576) NULL COMMENT "订单唯一键",
  `trade_no` varchar(1048576) NULL COMMENT "订单编号",
  `platform_id` varchar(1048576) NULL COMMENT "平台ID",
  `warehouse_type` varchar(1048576) NULL COMMENT "仓库类型",
  `src_tids` varchar(1048576) NULL COMMENT "原始单号",
  `trade_status` varchar(1048576) NULL COMMENT "订单状态",
  `trade_type` varchar(1048576) NULL COMMENT "订单类型",
  `delivery_term` varchar(1048576) NULL COMMENT "发货条件",
  `receiver_ring` varchar(1048576) NULL COMMENT "京东几环",
  `freeze_reason` varchar(1048576) NULL COMMENT "冻结原因",
  `refund_status` varchar(1048576) NULL COMMENT "退款状态",
  `fenxiao_type` varchar(1048576) NULL COMMENT "分销类别",
  `fenxiao_nick` varchar(1048576) NULL COMMENT "分销商昵称",
  `trade_time` varchar(1048576) NULL COMMENT "下单时间",
  `pay_time` varchar(1048576) NULL COMMENT "支付时间",
  `consign_time` varchar(1048576) NULL COMMENT "发货时间",
  `receiver_province` varchar(1048576) NULL COMMENT "省份id",
  `receiver_city` varchar(1048576) NULL COMMENT "城市id",
  `receiver_district` varchar(1048576) NULL COMMENT "地区id",
  `receiver_zip` varchar(1048576) NULL COMMENT "邮编",
  `receiver_area` varchar(1048576) NULL COMMENT "地区",
  `receiver_dtb` varchar(1048576) NULL COMMENT "大头笔",
  `bad_reason` varchar(1048576) NULL COMMENT "异常订单原因",
  `logistics_no` varchar(1048576) NULL COMMENT "物流单号",
  `buyer_message` varchar(1048576) NULL COMMENT "买家留言",
  `cs_remark` varchar(1048576) NULL COMMENT "客服备注",
  `remark_flag` varchar(1048576) NULL COMMENT "标旗",
  `print_remark` varchar(1048576) NULL COMMENT "打印备注",
  `goods_type_count` varchar(1048576) NULL COMMENT "货品种类数",
  `goods_count` varchar(1048576) NULL COMMENT "货品总量",
  `goods_amount` varchar(1048576) NULL COMMENT "总货款",
  `post_amount` varchar(1048576) NULL COMMENT "邮费",
  `other_amount` varchar(1048576) NULL COMMENT "其他收费",
  `discount` varchar(1048576) NULL COMMENT "优惠",
  `receivable` varchar(1048576) NULL COMMENT "应收",
  `cod_amount` varchar(1048576) NULL COMMENT "货到付款金额",
  `ext_cod_fee` varchar(1048576) NULL COMMENT "买家COD费用",
  `goods_cost` varchar(1048576) NULL COMMENT "预估货品成本",
  `post_cost` varchar(1048576) NULL COMMENT "预估邮资成本",
  `weight` varchar(1048576) NULL COMMENT "预估重量",
  `profit` varchar(1048576) NULL COMMENT "预估毛利",
  `tax` varchar(1048576) NULL COMMENT "税额",
  `tax_rate` varchar(1048576) NULL COMMENT "税率",
  `commission` varchar(1048576) NULL COMMENT "佣金",
  `invoice_type` varchar(1048576) NULL COMMENT "发票类型",
  `invoice_title` varchar(1048576) NULL COMMENT "发票抬头",
  `invoice_content` varchar(1048576) NULL COMMENT "发票内容",
  `salesman_name` varchar(1048576) NULL COMMENT "业务员",
  `checker_name` varchar(1048576) NULL COMMENT "审核人",
  `fchecker_name` varchar(1048576) NULL COMMENT "财审人",
  `checkouter_name` varchar(1048576) NULL COMMENT "签出人",
  `stockout_no` varchar(1048576) NULL COMMENT "出库单号",
  `flag_name` varchar(1048576) NULL COMMENT "标记名称",
  `trade_from` varchar(1048576) NULL COMMENT "订单来源",
  `single_spec_no` varchar(1048576) NULL COMMENT "货品商家编码",
  `raw_goods_count` varchar(1048576) NULL COMMENT "原始货品数量",
  `raw_goods_type_count` varchar(1048576) NULL COMMENT "原始货品种类数",
  `currency` varchar(1048576) NULL COMMENT "币种",
  `invoice_id` varchar(1048576) NULL COMMENT "发票ID",
  `version_id` varchar(1048576) NULL COMMENT "版本号",
  `modified` varchar(1048576) NULL COMMENT "修改时间",
  `created` varchar(1048576) NULL COMMENT "创建时间",
  `check_time` varchar(1048576) NULL COMMENT "审核时间",
  `id_card_type` varchar(1048576) NULL COMMENT "证件类别",
  `shop_no` varchar(1048576) NULL COMMENT "店铺编号",
  `shop_name` varchar(1048576) NULL COMMENT "店铺名称",
  `shop_remark` varchar(1048576) NULL COMMENT "店铺备注",
  `warehouse_no` varchar(1048576) NULL COMMENT "仓库编号",
  `customer_no` varchar(1048576) NULL COMMENT "客户编码",
  `logistics_name` varchar(1048576) NULL COMMENT "物流名称",
  `logistics_code` varchar(1048576) NULL COMMENT "物流公司编号",
  `logistics_type_name` varchar(1048576) NULL COMMENT "物流类型",
  `to_deliver_time` varchar(1048576) NULL COMMENT "送货时间",
  `delay_to_time` varchar(1048576) NULL COMMENT "预计发货时间",
  `estimate_consign_time` varchar(1048576) NULL COMMENT "最晚发货时间",
  `shop_id` varchar(1048576) NULL COMMENT "店铺id",
  `warehouse_id` varchar(1048576) NULL COMMENT "仓库id",
  `trade_label` varchar(1048576) NULL COMMENT "订单标签",
  `trade_mask` varchar(1048576) NULL COMMENT "订单掩码",
  `shop_platform_id` varchar(1048576) NULL COMMENT "店铺平台id",
  `sub_platform_id` varchar(1048576) NULL COMMENT "子平台id",
  `package_name` varchar(1048576) NULL COMMENT "包装",
  `package_id` varchar(1048576) NULL COMMENT "包装id",
  `paid` varchar(1048576) NULL COMMENT "已付",
  `large_type` varchar(1048576) NULL COMMENT "大件类型",
  `gift_mask` varchar(1048576) NULL COMMENT "赠品标记",
  `customer_id` varchar(1048576) NULL COMMENT "客户id",
  `other_cost` varchar(1048576) NULL COMMENT "其他成本",
  `is_sealed` varchar(1048576) NULL COMMENT "不可合并拆分",
  `customer_type` varchar(1048576) NULL COMMENT "客户类型",
  `logistics_id` varchar(1048576) NULL COMMENT "物流公司id",
  `cancel_reason` varchar(1048576) NULL COMMENT "取消原因",
  `revert_reason` varchar(1048576) NULL COMMENT "驳回原因",
  `detail_list` json NULL COMMENT "订单明细",
  `origin_data` varchar(1048576) NULL COMMENT "原始数据",
  `yuce_cube_shop_id` varchar(1048576) NULL COMMENT "魔方店铺标识"
) ENGINE=OLAP 
PRIMARY KEY(`dt`, `__id`)
COMMENT "ODS_销售订单信息表(旺店通旗舰奇门API)"
PARTITION BY date_trunc('day', dt)
DISTRIBUTED BY HASH(`dt`, `__id`)
PROPERTIES (
"compression" = "LZ4",
"enable_persistent_index" = "true",
"fast_schema_evolution" = "true",
"replicated_storage" = "true",
"replication_num" = "3"
);

target schema:

CREATE TABLE `tmp_ods_api_wdtqjqm_sale_order_info_f_iceberg_test2` (
  `__id` bigint(20) DEFAULT NULL COMMENT "__id",
  `trade_id` varchar(1073741824) DEFAULT NULL COMMENT "订单唯一键",
  `trade_no` varchar(1073741824) DEFAULT NULL COMMENT "订单编号",
  `platform_id` varchar(1073741824) DEFAULT NULL COMMENT "平台ID",
  `warehouse_type` varchar(1073741824) DEFAULT NULL COMMENT "仓库类型",
  `src_tids` varchar(1073741824) DEFAULT NULL COMMENT "原始单号",
  `trade_status` varchar(1073741824) DEFAULT NULL COMMENT "订单状态",
  `trade_type` varchar(1073741824) DEFAULT NULL COMMENT "订单类型",
  `delivery_term` varchar(1073741824) DEFAULT NULL COMMENT "发货条件",
  `receiver_ring` varchar(1073741824) DEFAULT NULL COMMENT "京东几环",
  `freeze_reason` varchar(1073741824) DEFAULT NULL COMMENT "冻结原因",
  `refund_status` varchar(1073741824) DEFAULT NULL COMMENT "退款状态",
  `fenxiao_type` varchar(1073741824) DEFAULT NULL COMMENT "分销类别",
  `fenxiao_nick` varchar(1073741824) DEFAULT NULL COMMENT "分销商昵称",
  `trade_time` varchar(1073741824) DEFAULT NULL COMMENT "下单时间",
  `pay_time` varchar(1073741824) DEFAULT NULL COMMENT "支付时间",
  `consign_time` varchar(1073741824) DEFAULT NULL COMMENT "发货时间",
  `receiver_province` varchar(1073741824) DEFAULT NULL COMMENT "省份id",
  `receiver_city` varchar(1073741824) DEFAULT NULL COMMENT "城市id",
  `receiver_district` varchar(1073741824) DEFAULT NULL COMMENT "地区id",
  `receiver_zip` varchar(1073741824) DEFAULT NULL COMMENT "邮编",
  `receiver_area` varchar(1073741824) DEFAULT NULL COMMENT "地区",
  `receiver_dtb` varchar(1073741824) DEFAULT NULL COMMENT "大头笔",
  `bad_reason` varchar(1073741824) DEFAULT NULL COMMENT "异常订单原因",
  `logistics_no` varchar(1073741824) DEFAULT NULL COMMENT "物流单号",
  `buyer_message` varchar(1073741824) DEFAULT NULL COMMENT "买家留言",
  `cs_remark` varchar(1073741824) DEFAULT NULL COMMENT "客服备注",
  `remark_flag` varchar(1073741824) DEFAULT NULL COMMENT "标旗",
  `print_remark` varchar(1073741824) DEFAULT NULL COMMENT "打印备注",
  `goods_type_count` varchar(1073741824) DEFAULT NULL COMMENT "货品种类数",
  `goods_count` varchar(1073741824) DEFAULT NULL COMMENT "货品总量",
  `goods_amount` varchar(1073741824) DEFAULT NULL COMMENT "总货款",
  `post_amount` varchar(1073741824) DEFAULT NULL COMMENT "邮费",
  `other_amount` varchar(1073741824) DEFAULT NULL COMMENT "其他收费",
  `discount` varchar(1073741824) DEFAULT NULL COMMENT "优惠",
  `receivable` varchar(1073741824) DEFAULT NULL COMMENT "应收",
  `cod_amount` varchar(1073741824) DEFAULT NULL COMMENT "货到付款金额",
  `ext_cod_fee` varchar(1073741824) DEFAULT NULL COMMENT "买家COD费用",
  `goods_cost` varchar(1073741824) DEFAULT NULL COMMENT "预估货品成本",
  `post_cost` varchar(1073741824) DEFAULT NULL COMMENT "预估邮资成本",
  `weight` varchar(1073741824) DEFAULT NULL COMMENT "预估重量",
  `profit` varchar(1073741824) DEFAULT NULL COMMENT "预估毛利",
  `tax` varchar(1073741824) DEFAULT NULL COMMENT "税额",
  `tax_rate` varchar(1073741824) DEFAULT NULL COMMENT "税率",
  `commission` varchar(1073741824) DEFAULT NULL COMMENT "佣金",
  `invoice_type` varchar(1073741824) DEFAULT NULL COMMENT "发票类型",
  `invoice_title` varchar(1073741824) DEFAULT NULL COMMENT "发票抬头",
  `invoice_content` varchar(1073741824) DEFAULT NULL COMMENT "发票内容",
  `salesman_name` varchar(1073741824) DEFAULT NULL COMMENT "业务员",
  `checker_name` varchar(1073741824) DEFAULT NULL COMMENT "审核人",
  `fchecker_name` varchar(1073741824) DEFAULT NULL COMMENT "财审人",
  `checkouter_name` varchar(1073741824) DEFAULT NULL COMMENT "签出人",
  `stockout_no` varchar(1073741824) DEFAULT NULL COMMENT "出库单号",
  `flag_name` varchar(1073741824) DEFAULT NULL COMMENT "标记名称",
  `trade_from` varchar(1073741824) DEFAULT NULL COMMENT "订单来源",
  `single_spec_no` varchar(1073741824) DEFAULT NULL COMMENT "货品商家编码",
  `raw_goods_count` varchar(1073741824) DEFAULT NULL COMMENT "原始货品数量",
  `raw_goods_type_count` varchar(1073741824) DEFAULT NULL COMMENT "原始货品种类数",
  `currency` varchar(1073741824) DEFAULT NULL COMMENT "币种",
  `invoice_id` varchar(1073741824) DEFAULT NULL COMMENT "发票ID",
  `version_id` varchar(1073741824) DEFAULT NULL COMMENT "版本号",
  `modified` varchar(1073741824) DEFAULT NULL COMMENT "修改时间",
  `created` varchar(1073741824) DEFAULT NULL COMMENT "创建时间",
  `check_time` varchar(1073741824) DEFAULT NULL COMMENT "审核时间",
  `id_card_type` varchar(1073741824) DEFAULT NULL COMMENT "证件类别",
  `shop_no` varchar(1073741824) DEFAULT NULL COMMENT "店铺编号",
  `shop_name` varchar(1073741824) DEFAULT NULL COMMENT "店铺名称",
  `shop_remark` varchar(1073741824) DEFAULT NULL COMMENT "店铺备注",
  `warehouse_no` varchar(1073741824) DEFAULT NULL COMMENT "仓库编号",
  `customer_no` varchar(1073741824) DEFAULT NULL COMMENT "客户编码",
  `logistics_name` varchar(1073741824) DEFAULT NULL COMMENT "物流名称",
  `logistics_code` varchar(1073741824) DEFAULT NULL COMMENT "物流公司编号",
  `logistics_type_name` varchar(1073741824) DEFAULT NULL COMMENT "物流类型",
  `to_deliver_time` varchar(1073741824) DEFAULT NULL COMMENT "送货时间",
  `delay_to_time` varchar(1073741824) DEFAULT NULL COMMENT "预计发货时间",
  `estimate_consign_time` varchar(1073741824) DEFAULT NULL COMMENT "最晚发货时间",
  `shop_id` varchar(1073741824) DEFAULT NULL COMMENT "店铺id",
  `warehouse_id` varchar(1073741824) DEFAULT NULL COMMENT "仓库id",
  `trade_label` varchar(1073741824) DEFAULT NULL COMMENT "订单标签",
  `trade_mask` varchar(1073741824) DEFAULT NULL COMMENT "订单掩码",
  `shop_platform_id` varchar(1073741824) DEFAULT NULL COMMENT "店铺平台id",
  `sub_platform_id` varchar(1073741824) DEFAULT NULL COMMENT "子平台id",
  `package_name` varchar(1073741824) DEFAULT NULL COMMENT "包装",
  `package_id` varchar(1073741824) DEFAULT NULL COMMENT "包装id",
  `paid` varchar(1073741824) DEFAULT NULL COMMENT "已付",
  `large_type` varchar(1073741824) DEFAULT NULL COMMENT "大件类型",
  `gift_mask` varchar(1073741824) DEFAULT NULL COMMENT "赠品标记",
  `customer_id` varchar(1073741824) DEFAULT NULL COMMENT "客户id",
  `other_cost` varchar(1073741824) DEFAULT NULL COMMENT "其他成本",
  `is_sealed` varchar(1073741824) DEFAULT NULL COMMENT "不可合并拆分",
  `customer_type` varchar(1073741824) DEFAULT NULL COMMENT "客户类型",
  `logistics_id` varchar(1073741824) DEFAULT NULL COMMENT "物流公司id",
  `cancel_reason` varchar(1073741824) DEFAULT NULL COMMENT "取消原因",
  `revert_reason` varchar(1073741824) DEFAULT NULL COMMENT "驳回原因",
  `detail_list` varchar(1073741824) DEFAULT NULL COMMENT "订单明细",
  `origin_data` varchar(1073741824) DEFAULT NULL COMMENT "原始数据",
  `yuce_cube_shop_id` varchar(1073741824) DEFAULT NULL COMMENT "魔方店铺标识",
  `dt` varchar(1073741824) DEFAULT NULL COMMENT "日分区"
)
PARTITION BY (dt)
PROPERTIES ("location" = "oss://backet/datalake/test/tmp_ods_api_wdtqjqm_sale_order_info_f_iceberg_test2")

query涉及的分区数
746个分区

image5120×1038 494 KB

query:

insert into iceberg.test_db.tmp_ods_api_wdtqjqm_sale_order_info_f_iceberg_test2 select * from default_catalog.internal.ods_api_wdtqjqm_sale_order_info_f

colagy · 2024年11月28日 11:38

大佬可不可以有个配置文件可以配置数据湖写入数据时的内存占用大小

许秀不许秀 · 2024年12月6日 08:06

先限制下单次写入分区数量