kafka_3.7.0(sasl+acl)+管理工具redpanda

一、环境说明

二、安装docker
略。。。

三、安装kafka

path=/data/kafka
mkdir -p ${path}/{data,etc,log}
chown -R 5000 ${path}

cat >${path}/etc/sasl_config.properties<< 'EOF'
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="gohangout" password="Gohangout#XXXX";
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
EOF

# KAFKA_NODE_ID、 KAFKA_CFG_ADVERTISED_LISTENERS  、 KAFKA_CFG_CONTROLLER_QUORUM_VOTERS 根据实际情况填写
cat > ${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`

docker rm -f kafka

docker run -d \
--name kafka \
--restart=always \
--net host \
--user 5000 \
--add-host=logaudit_kafka_01:192.168.11.100 \
--add-host=logaudit_kafka_02:192.168.11.101 \
--add-host=logaudit_kafka_03:192.168.11.102 \
-e KAFKA_NODE_ID=1 \
-e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://logaudit_kafka_01:9092 \
-e KAFKA_DAEMON_USER=5000 \
-e KAFKA_DAEMON_GROUP=5000 \
-e KAFKA_HEAP_OPTS="-Xmx512m -Xms512m" \
-e KAFKA_CFG_PROCESS_ROLES=broker,controller \
-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN \
-e KAFKA_CONTROLLER_USER=contr0ller \
-e KAFKA_CONTROLLER_PASSWORD=Contr0ller#XXXX \
-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:SASL_PLAINTEXT,CONTROLLER:SASL_PLAINTEXT \
-e KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_ENABLE_KRAFT=yes \
-e KAFKA_KRAFT_CLUSTER_ID="Aqvf7RVETX-DInZbNUXXXXXXX" \
-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@192.168.11.100:9093,2@192.168.11.101:9093,3@192.168.11.102:9093 \
-e ALLOW_PLAINTEXT_LISTENER=yes \
-e KAFKA_TLS_CLIENT_AUTH=none \
-e KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN \
-e KAFKA_CLIENT_LISTENER_NAME=PLAINTEXT \
-e KAFKA_CLIENT_USERS=gohangout \
-e KAFKA_CLIENT_PASSWORDS=Gohangout#XXXX \
-e KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true \
-e KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 \
-e KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND=true \
-e KAFKA_CFG_SUPER_USERS=User:gohangout \
-e KAFKA_CFG_AUTHORIZER_CLASS_NAME=org.apache.kafka.metadata.authorizer.StandardAuthorizer \
-v `pwd`/etc/sasl_config.properties:/opt/bitnami/kafka/config/sasl_config.properties \
-v `pwd`/data:/bitnami/kafka/ \
-v /etc/localtime:/etc/localtime \
bitnami/kafka:3.7.0
EOF

bash ${path}/start.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

四、 kafka测试

docker exec -it kafka bash 

#创建topic
kafka-topics.sh --create --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test  --command-config /opt/bitnami/kafka/config/sasl_config.properties



#生产
kafka-console-producer.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test --producer.config /opt/bitnami/kafka/config/sasl_config.properties
#消费
kafka-console-consumer.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test --consumer.config /opt/bitnami/kafka/config/sasl_config.properties



#扩容分区
kafka-topics.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --alter --topic  log-smartgate --partitions 3 --replication-factor 1 --command-config /opt/bitnami/kafka/config/sasl_config.properties

#查询分区
kafka-topics.sh --describe --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test  --command-config /opt/bitnami/kafka/config/sasl_config.properties
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

#acl ,需要在添加启动参数
kafka-acls.sh  --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --add --allow-principal User:gohangout --operation ALL --topic test --command-config /opt/bitnami/kafka/config/sasl_config.properties

Adding ACLs for resource `ResourcePattern(resourceType=TOPIC, name=test, patternType=LITERAL)`:
        (principal=User:gohangout, host=*, operation=ALL, permissionType=ALLOW)

Current ACLs for resource `ResourcePattern(resourceType=TOPIC, name=test, patternType=LITERAL)`:
        (principal=User:gohangout, host=*, operation=ALL, permissionType=ALLOW)

#这三台配置是与ALC有关
#-e KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND=true \
#-e KAFKA_CFG_SUPER_USERS=User:gohangout \
#-e KAFKA_CFG_AUTHORIZER_CLASS_NAME=org.apache.kafka.metadata.authorizer.StandardAuthorizer \

#ACL有以下权限
Describe
DescribeConfigs
Alter
IdempotentWrite
Read
Delete
Create
ClusterAction
All
CreateTokens
DescribeTokens
Write
AlterConfigs

#查看ACL授权
kafka-acls.sh  --bootstrap-server 192.168.11.103:9092 --list --command-config /opt/bitnami/kafka/config/sasl_config.properties

Current ACLs for resource `ResourcePattern(resourceType=TOPIC, name=test, patternType=LITERAL)`:
        (principal=User:gohangout, host=*, operation=ALL, permissionType=ALLOW)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

四、安装redpanda

path=/data/kowl
mkdir -p ${path}/etc

cat > ${path}/etc/console.yaml << 'EOF'
server:
#   listenAddress:
  listenPort: 19002
logger:
  level: info
analytics:
  enabled: false
EOF

cat > ${path}/start.sh << 'EOF'
docker rm -f kowl

cd $(dirname $0)
docker run -itd \
--restart=always \
--network host \
--name kowl \
--user 5000 \
--add-host=logaudit_kafka_01:192.168.11.100 \
--add-host=logaudit_kafka_02:192.168.11.101 \
--add-host=logaudit_kafka_03:192.168.11.102 \
-v /etc/localtime:/etc/localtime \
-v `pwd`/etc/console.yaml:/app/console.yaml \
-e KAFKA_BROKERS="logaudit_kafka_01:9092,logaudit_kafka_02:9092,logaudit_kafka_03:9092" \
-e KAFKA_TLS_ENABLED=false \
-e KAFKA_SASL_ENABLED=true \
-e KAFKA_SASL_USERNAME=gohangout \
-e KAFKA_SASL_PASSWORD="Gohangout#XXXX" \
redpandadata/console:v2.4.5 \
-config.filepath /app/console.yaml

EOF

bash ${path}/start.sh 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38