【存算分离】es catalog无法连接上腾讯云的es集群

💬 StarRocks 用户问答 功能使用相关
#1

pv0fiP
pv0fiP2726×1200 214 KB

本地验证过存算一体的starrocks 连接本地自己搭建的es可以，连接腾讯云es集群也不行。

  • starrocks 集群情况
    版本：3.2.2
    存算分离 shared_data mode
    搭建过程

  • 验证es是否可连接

curl --user elastic:XXX -XGET 'http://172.16.0.8:9200/_nodes/http?pretty'
{
  "_nodes" : {
    "total" : 2,
    "successful" : 2,
    "failed" : 0
  },
  "cluster_name" : "es-n29hodty",
  "nodes" : {
    "8JMmm6BmTti1NkIK5klb9Q" : {
      "name" : "1659921770001140232",
      "transport_address" : "172.16.0.7:9300",
      "host" : "172.16.0.7",
      "ip" : "172.16.0.7",
      "version" : "7.14.2",
      "build_flavor" : "default",
      "build_type" : "tar",
      "build_hash" : "6a9d8f44123874a3e089149a7e70bca7c8474e58",
      "roles" : [
        "data_content",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes" : {
        "ml.machine_memory" : "3929686016",
        "rack" : "cvm_1_100003",
        "xpack.installed" : "true",
        "set" : "100003",
        "transform.node" : "true",
        "ip" : "9.10.119.35",
        "temperature" : "warm",
        "ml.max_open_jobs" : "512",
        "ml.max_jvm_size" : "1610612736",
        "region" : "1"
      },
      "http" : {
        "bound_address" : [
          "[::]:9200"
        ],
        "publish_address" : "172.16.0.7:9200",
        "max_content_length_in_bytes" : 104857600
      }
    },
    "ra5MMv2SRuWmQxajPEwLBw" : {
      "name" : "1659921770001140132",
      "transport_address" : "172.16.0.12:9300",
      "host" : "172.16.0.12",
      "ip" : "172.16.0.12",
      "version" : "7.14.2",
      "build_flavor" : "default",
      "build_type" : "tar",
      "build_hash" : "6a9d8f44123874a3e089149a7e70bca7c8474e58",
      "roles" : [
        "data_content",
        "data_warm",
        "ingest",
        "master",
        "ml",
        "remote_cluster_client",
        "transform"
      ],
      "attributes" : {
        "ml.machine_memory" : "3929686016",
        "rack" : "cvm_1_100003",
        "xpack.installed" : "true",
        "set" : "100003",
        "transform.node" : "true",
        "ip" : "9.10.120.236",
        "temperature" : "warm",
        "ml.max_open_jobs" : "512",
        "ml.max_jvm_size" : "1610612736",
        "region" : "1"
      },
      "http" : {
        "bound_address" : [
          "[::]:9200"
        ],
        "publish_address" : "172.16.0.12:9200",
        "max_content_length_in_bytes" : 104857600
      }
    }
  }
}
  • 创建es catalog
CREATE EXTERNAL CATALOG es
PROPERTIES
(
    "type" = "es",
    "es.type" = "_doc",
    "hosts" = "http://172.16.0.8:9200",
    "es.net.ssl" = "true",
    "user" = "elastic",
    "password" = "XXX",
    "es.nodes.wan.only" = "true"
);
  • 查询es，无法查询出来tables
StarRocks > show catalogs;
+-----------------+----------+------------------------------------------------------------------+
| Catalog         | Type     | Comment                                                          |
+-----------------+----------+------------------------------------------------------------------+
| default_catalog | Internal | An internal catalog contains this cluster's self-managed tables. |
| es              | Es       | NULL                                                             |
| jdbc            | Jdbc     | NULL                                                             |
+-----------------+----------+------------------------------------------------------------------+

StarRocks > use es.defalut_db;
Database changed
StarRocks > show tables;
Empty set (0.05 sec)

而实际es内是有索引的

curl --user elastic:XXX -XGET 'http://172.16.0.8:9200/_cat/indices?pretty'
green open customers_cdc_v2                           exp_KAV8SLmnD2e6rdLUYg 5 1   88076  14576 164.7mb  81.2mb
green open customers_cdc_v1                           AcZHwEnCQBid6V9bD9-xJw 5 1   85931   1410 144.4mb  71.8mb
green open customers_cdc_v4                           hEUKge1CR1a6KaCHB45NUw 5 1   91832   8763 162.2mb  81.1mb
green open test                                       GjmL_rGLTaa9YmH2d-3PsQ 1 0       5      0     5kb     5kb
#2

show all grants看下

#3 
mysql> SHOW GRANTS;
+--------------+---------+----------------------------+
| UserIdentity | Catalog | Grants                     |
+--------------+---------+----------------------------+
| 'root'@'%'   | NULL    | GRANT 'root' TO 'root'@'%' |
+--------------+---------+----------------------------+
1 row in set (0.02 sec)

mysql> SHOW ALL GRANTS;
ERROR 1064 (HY000): Getting syntax error at line 1, column 9. Detail message: No viable statement for input 'SHOW ALL GRANTS'.
mysql>
#4

腾讯云那边的排查

image
image2418×1142 397 KB

image
image1774×1084 225 KB

image
image1796×736 83.2 KB

#5

image
image1524×448 60 KB

#6

es 里面的 index mapping 是怎么样的

#7

本地验证情况
docker临时部署 存算一体+ 自建es
线上的是 存算分离+腾讯云es产品
本地可以，线上不行

StarRocks > CREATE EXTERNAL CATALOG es
    -> PROPERTIES
    -> (
    ->     "type" = "es",
    ->     "es.type" = "_doc",
    ->     "hosts" = "http://192.168.103.113:9200",
    ->     "es.net.ssl" = "true",
    ->     "user" = "elastic",
    ->     "password" = "@dakewe2022",
    ->     "es.nodes.wan.only" = "false"
    -> );
Query OK, 0 rows affected (0.30 sec)

StarRocks > show catalogs;
+-----------------+----------+------------------------------------------------------------------+
| Catalog         | Type     | Comment                                                          |
+-----------------+----------+------------------------------------------------------------------+
| default_catalog | Internal | An internal catalog contains this cluster's self-managed tables. |
| es              | Es       | NULL                                                             |
+-----------------+----------+------------------------------------------------------------------+
2 rows in set (0.01 sec)

StarRocks > use es.default_db;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
StarRocks > show tables;
+----------------------------+
| Tables_in_default_db       |
+----------------------------+
| companies                  |
| customers                  |
| materials                  |
| materials_cdc_v7           |
| materials_cdc_v8           |
| monstache.stats.2023-07-12 |
| monstache.stats.2023-07-13 |
| operation_logs             |
| orders                     |
| sale_product_sink          |
+----------------------------+
10 rows in set (1.18 sec)
#8

跟index mapping好像都没关系了吧，是show tables就出不来，更别提查索引

#9

本地存算一体连接腾讯云ES产品，:x:

image
image1738×954 80.7 KB

#10

show table 本质就是把 index 转换成 table。
把 es.type 这一行移除了，建一个 catalog 看看？

#12

image
image1840×986 78 KB

删除了 es.type,也是不行

#13

fe.log 里面的日志还是一样报错？

#14

整个过程 fe没有产生错误日志

#15

腾讯云技术人员那边建了一个测试腾讯云es集群，也是添加不进去，应该不是index mapping的问题
如果是index mapping的问题，腾讯云那边应该也是要能show tables出来

#16

最后排查出来的问题：

云上的索引， GET index/_mapping 返回的mapping中有动态模板，自建的是没有的，动态模板是es原生的功能，自建的集群也可以设置动态模板，因此sr没有兼容这种情况

云上默认开启了动态模版，自建默认没有开启，sr解析动态模版是类型转换异常了
自建es开起动态模板 也会显示空

sr这里得兼容下：

image
image2148×1444 407 KB

索引验证无法show tables;

curl  -H "Content-type: application/json" -X PUT localhost:9200/sr_test -d '
{
    "mappings" : {
      "dynamic_templates" : [
        {
          "message_full" : {
            "match" : "message_full",
            "mapping" : {
              "fields" : {
                "keyword" : {
                  "ignore_above" : 2048,
                  "type" : "keyword"
                }
              },
              "type" : "text"
            }
          }
        },
        {
          "message" : {
            "match" : "message",
            "mapping" : {
              "type" : "text"
            }
          }
        },
        {
          "strings" : {
            "match_mapping_type" : "string",
            "mapping" : {
              "type" : "keyword"
            }
          }
        }
      ],
      "properties" : {
        "age" : {
          "type" : "keyword"
        },
        "name" : {
          "type" : "keyword"
        }
      }
    }
}'

查了下云es有一个这个mappints

{
  "mappings": {
    "properties": {
      "attributes": {
        "type": "keyword"
      },
      "cat": {
        "type": "keyword"
      },
      "inventory": {
        "dynamic": "true",   //---------> 这里
        "properties": {
          "BJ001": {
            "type": "long"
          },
          "BJ008": {
            "type": "long"
          }
        }
      },
    }
  },
  "settings": {
    "index.analysis.analyzer.ngram_lower.filter": [
      "lowercase"
    ],
    "index.analysis.analyzer.ngram_lower.tokenizer": "ngram_tokenizer",
    "index.analysis.tokenizer.ngram_tokenizer.custom_token_chars": [
      "-"
    ],
    "index.analysis.tokenizer.ngram_tokenizer.max_gram": "30",
    "index.analysis.tokenizer.ngram_tokenizer.min_gram": "1",
    "index.analysis.tokenizer.ngram_tokenizer.token_chars": [
      "letter",
      "digit",
      "custom"
    ],
    "index.analysis.tokenizer.ngram_tokenizer.type": "ngram",
    "index.creation_date": "1698827510776",
    "index.max_ngram_diff": "30",
    "index.max_result_window": "500000",
    "index.number_of_replicas": "1",
    "index.number_of_shards": "5",
    "index.provided_name": "materials_cdc_v8",
    "index.refresh_interval": "30s",
    "index.routing.allocation.require.temperature": "warm",
    "index.translog.durability": "async",
    "index.translog.sync_interval": "5s",
    "index.unassigned.node_left.delayed_timeout": "5m",
    "index.uuid": "KfwPEdSwS5SBtYPfkHRc2g",
    "index.version.created": "7140299"
  }
}
#17

看起来是的，你有兴趣 fix 它吗

#18

doris 好像已经修复了

#19

请问您后续找到解决办法了么，求教

京ICP备2022026421号-1