如何解决使用 docker-compose 将 mysql 添加到 Apache-Druid 作为元数据存储
有没有人能够使用 docker-compose 文件将 MysqL 用于 Druid 的元数据？我已经花了几个小时，但我仍然在 druid broker 组件中遇到错误。
我想使用 MysqL 而不是 postgres 的原因是因为供应商的 docker 容器错误地连接到 Druid 的 postgres 服务器而不是它自己的并且更改 postgres 的端口没有帮助。
以下是我在“经纪人”中看到的错误。
2021-02-23T15:15:22,464 DEBUG [qtp575779772-162] org.apache.druid.jetty.RequestLog - 172.16.1.16 POST //172.16.1.13:8082/druid/v2/sql HTTP/1.1
2021-02-23T15:15:23,063 ERROR [sql[402982ef-7105-490a-83d5-8528580c36d3]] org.apache.druid.curator.discovery.ServerdiscoverySelector - No server instance found for [druid/overlord]
2021-02-23T15:15:23,063 ERROR [sql[18a217cf-13a1-444b-99b2-726a55152af1]] org.apache.druid.curator.discovery.ServerdiscoverySelector - No server instance found for [druid/coordinator]
2021-02-23T15:15:23,063 ERROR [sql[2e472014-8050-4886-81e4-4583fa601b40]] org.apache.druid.curator.discovery.ServerdiscoverySelector - No server instance found for [druid/coordinator]
2021-02-23T15:15:23,063 ERROR [sql[41ce0e01-9e49-40ae-a7df-bc01a1c0b11c]] org.apache.druid.curator.discovery.ServerdiscoverySelector - No server instance found for [druid/overlord]
2021-02-23T15:15:23,072 INFO [NodeRoleWatcher[HISTORICAL]] org.apache.druid.curator.discovery.CuratorDruidNodediscoveryProvider$NodeRoleWatcher - Node[http://172.16.1.14:8083] of role[historical] detected.
2021-02-23T15:15:23,073 INFO [NodeRoleWatcher[HISTORICAL]] org.apache.druid.curator.discovery.CuratorDruidNodediscoveryProvider$NodeRoleWatcher - Node watcher of role[historical] is Now initialized.
2021-02-23T15:15:23,064 WARN [sql[18a217cf-13a1-444b-99b2-726a55152af1]] org.apache.druid.sql.http.sqlResource - Failed to handle query: sqlQuery{query='SELECT datasource FROM sys.segments GROUP BY 1',resultFormat=OBJECT,header=false,context={},parameters=[]}
java.lang.RuntimeException: org.apache.druid.java.util.common.IOE: No kNown server
at org.apache.druid.sql.calcite.schema.SystemSchema.getThingsFromleaderNode(SystemSchema.java:1008) ~[druid-sql-0.20.0.jar:0.20.0]
解决方法
我找到了错误详情。它在德鲁伊协调员日志中。根据以下链接，MySql JDBC jar 文件未随 Druid docker 映像一起提供。 https://druid.apache.org/docs/latest/development/extensions-core/mysql.html
所以我不得不将它复制到我使用元数据存储的 Druid 容器中。在那之后，一切都像魅力一样。
以下是我正在使用的 docker-compose 文件和环境文件中的详细信息：
docker-compose.yml
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License,Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,# software distributed under the License is distributed on an
# "AS IS" BASIS,WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND,either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
version: "2.2"
volumes:
metadata_data: {}
middle_var: {}
historical_var: {}
broker_var: {}
coordinator_var: {}
router_var: {}
mysql-data: {}
services:
mysql:
image: mysql
restart: always
environment:
- MYSQL_RANDOM_ROOT_PASSWORD='yes'
- MYSQL_USER=druid
- MYSQL_PASSWORD=diurd
- MYSQL_DATABASE=druid
command:
- --character-set-server=utf8
- --collation-server=utf8_unicode_ci
volumes:
- 'mysql-data:/var/lib/mysql:rw'
networks:
cc_subnet:
ipv4_address: 172.16.1.10
# Need 3.5 or later for container nodes
druid_zookeeper:
container_name: zookeeper
hostname: druid_zookeeper
image: zookeeper:3.5
environment:
- ZOO_MY_ID=1
networks:
cc_subnet:
ipv4_address: 172.16.1.11
coordinator:
image: apache/druid:0.20.0
container_name: coordinator
volumes:
- ./storage:/opt/data
- coordinator_var:/opt/druid/var
- ./extensions/mysql-connector-java-5.1.48.jar:/opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.48.jar
depends_on:
- druid_zookeeper
- mysql
ports:
- "8081:8081"
command:
- coordinator
env_file:
- environment
networks:
cc_subnet:
ipv4_address: 172.16.1.12
broker:
image: apache/druid:0.20.0
container_name: broker
volumes:
- broker_var:/opt/druid/var
- ./extensions/mysql-connector-java-5.1.48.jar:/opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.48.jar
depends_on:
- druid_zookeeper
- mysql
- coordinator
ports:
- "8082:8082"
command:
- broker
env_file:
- environment
networks:
cc_subnet:
ipv4_address: 172.16.1.13
historical:
image: apache/druid:0.20.0
container_name: historical
volumes:
- ./storage:/opt/data
- historical_var:/opt/druid/var
- ./extensions/mysql-connector-java-5.1.48.jar:/opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.48.jar
depends_on:
- druid_zookeeper
- mysql
- coordinator
ports:
- "8083:8083"
command:
- historical
env_file:
- environment
networks:
cc_subnet:
ipv4_address: 172.16.1.14
middlemanager:
image: apache/druid:0.20.0
container_name: middlemanager
volumes:
- ./storage:/opt/data
- middle_var:/opt/druid/var
- ./extensions/mysql-connector-java-5.1.48.jar:/opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.48.jar
depends_on:
- druid_zookeeper
- mysql
- coordinator
ports:
- "8091:8091"
command:
- middleManager
env_file:
- environment
networks:
cc_subnet:
ipv4_address: 172.16.1.15
router:
image: apache/druid:0.20.0
container_name: router
volumes:
- router_var:/opt/druid/var
- ./extensions/mysql-connector-java-5.1.48.jar:/opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.48.jar
depends_on:
- druid_zookeeper
- mysql
- coordinator
ports:
- "8888:8888"
command:
- router
env_file:
- environment
networks:
cc_subnet:
ipv4_address: 172.16.1.16
networks:
cc_subnet:
external: true
环境
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License,either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Java tuning
DRUID_XMX=1g
DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m
DRUID_MAXDIRECTMEMORYSIZE=6172m
druid_emitter_logging_logLevel=debug
druid_extensions_loadList=["druid-histogram","druid-datasketches","druid-lookups-cached-global","mysql-metadata-storage"]
druid_zk_service_host=druid_zookeeper
druid_metadata_storage_host=
druid_metadata_storage_type=mysql
druid_metadata_storage_connector_connectURI=jdbc:mysql://mysql:3306/druid?allowPublicKeyRetrieval=true&useSSL=false
druid_metadata_storage_connector_user=druid
druid_metadata_storage_connector_password=diurd
druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server","-Xmx1g","-Xms1g","-XX:MaxDirectMemorySize=3g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
druid_storage_type=local
druid_storage_storageDirectory=/opt/data/segments
druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/data/indexing-logs
druid_processing_numThreads=2
druid_processing_numMergeBuffers=2
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
