404 lines
14 KiB
YAML
404 lines
14 KiB
YAML
name: redpanda-quickstart-multi-broker
|
|
networks:
|
|
redpanda_network:
|
|
driver: bridge
|
|
volumes:
|
|
redpanda-0: null
|
|
redpanda-1: null
|
|
redpanda-2: null
|
|
minio: null
|
|
services:
|
|
##################
|
|
# Redpanda Brokers #
|
|
##################
|
|
redpanda-0:
|
|
command:
|
|
- redpanda
|
|
- start
|
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
|
# Address the broker advertises to clients that connect to the Kafka API.
|
|
# Use the internal addresses to connect to the Redpanda brokers
|
|
# from inside the same Docker network.
|
|
# Use the external addresses to connect to the Redpanda brokers
|
|
# from outside the Docker network.
|
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
|
- --rpc-addr redpanda-0:33145
|
|
- --advertise-rpc-addr redpanda-0:33145
|
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
|
- --mode dev-container
|
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
|
- --smp 1
|
|
- --default-log-level=info
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
container_name: redpanda-0
|
|
# Sets the username and password of the bootstrap SCRAM superuser
|
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
|
environment:
|
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
volumes:
|
|
- redpanda-0:/var/lib/redpanda/data
|
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
networks:
|
|
- redpanda_network
|
|
ports:
|
|
- 18081:18081
|
|
- 18082:18082
|
|
- 19092:19092
|
|
- 19644:9644
|
|
healthcheck:
|
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
|
interval: 10s
|
|
timeout: 15s
|
|
retries: 10
|
|
depends_on:
|
|
minio:
|
|
condition: service_healthy
|
|
redpanda-1:
|
|
command:
|
|
- redpanda
|
|
- start
|
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
|
- --rpc-addr redpanda-1:33145
|
|
- --advertise-rpc-addr redpanda-1:33145
|
|
- --mode dev-container
|
|
- --smp 1
|
|
- --default-log-level=info
|
|
- --seeds redpanda-0:33145
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
container_name: redpanda-1
|
|
environment:
|
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
volumes:
|
|
- redpanda-1:/var/lib/redpanda/data
|
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
networks:
|
|
- redpanda_network
|
|
ports:
|
|
- 28081:28081
|
|
- 28082:28082
|
|
- 29092:29092
|
|
- 29644:9644
|
|
depends_on:
|
|
- redpanda-0
|
|
- minio
|
|
redpanda-2:
|
|
command:
|
|
- redpanda
|
|
- start
|
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
|
- --rpc-addr redpanda-2:33145
|
|
- --advertise-rpc-addr redpanda-2:33145
|
|
- --mode dev-container
|
|
- --smp 1
|
|
- --default-log-level=info
|
|
- --seeds redpanda-0:33145
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
container_name: redpanda-2
|
|
environment:
|
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
volumes:
|
|
- redpanda-2:/var/lib/redpanda/data
|
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
networks:
|
|
- redpanda_network
|
|
ports:
|
|
- 38081:38081
|
|
- 38082:38082
|
|
- 39092:39092
|
|
- 39644:9644
|
|
depends_on:
|
|
- redpanda-0
|
|
- minio
|
|
####################
|
|
# Redpanda Console #
|
|
####################
|
|
console:
|
|
container_name: redpanda-console
|
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
|
networks:
|
|
- redpanda_network
|
|
entrypoint: /bin/sh
|
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
|
volumes:
|
|
- ./config:/tmp/config/
|
|
environment:
|
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
|
CONSOLE_CONFIG_FILE: |
|
|
# Configure a connection to the Redpanda cluster
|
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
|
kafka:
|
|
brokers: ["redpanda-0:9092"]
|
|
sasl:
|
|
enabled: true
|
|
impersonateUser: true
|
|
schemaRegistry:
|
|
enabled: true
|
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
|
authentication:
|
|
impersonateUser: true
|
|
redpanda:
|
|
adminApi:
|
|
enabled: true
|
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
|
authentication:
|
|
basic:
|
|
username: superuser
|
|
password: secretpassword
|
|
impersonateUser: false
|
|
console:
|
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
|
topicDocumentation:
|
|
enabled: true
|
|
git:
|
|
enabled: true
|
|
repository:
|
|
url: https://github.com/redpanda-data/docs
|
|
branch: main
|
|
baseDirectory: tests/docker-compose
|
|
authentication:
|
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
|
basic:
|
|
enabled: true
|
|
authorization:
|
|
roleBindings:
|
|
- roleName: admin
|
|
users:
|
|
- loginType: basic
|
|
name: superuser
|
|
ports:
|
|
- 8080:8080
|
|
depends_on:
|
|
redpanda-0:
|
|
condition: service_healthy
|
|
createtopic:
|
|
condition: service_completed_successfully
|
|
registerschema:
|
|
condition: service_completed_successfully
|
|
deploytransform:
|
|
condition: service_completed_successfully
|
|
####################
|
|
# Redpanda Connect #
|
|
####################
|
|
connect:
|
|
container_name: redpanda-connect
|
|
image: docker.redpanda.com/redpandadata/connect
|
|
networks:
|
|
- redpanda_network
|
|
entrypoint: /bin/sh
|
|
depends_on:
|
|
redpanda-0:
|
|
condition: service_healthy
|
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
|
environment:
|
|
# This Redpanda Connect configuration creates fake data,
|
|
# processes it, and writes the output to a set of topics.
|
|
#
|
|
# Input:
|
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
|
# Pipeline:
|
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
|
# - Unarchive processor to parse the JSON array and extract each
|
|
# element into its own message.
|
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
|
# Output:
|
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
|
CONNECT_CFG_FILE: |
|
|
input:
|
|
generate:
|
|
interval: 1s
|
|
mapping: |
|
|
let first_name = fake("first_name")
|
|
let last_name = fake("last_name")
|
|
|
|
root.user_id = counter()
|
|
root.name = $$first_name + " " + $$last_name
|
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
|
root.ip = fake("ipv4")
|
|
root.login_time = now()
|
|
pipeline:
|
|
processors:
|
|
- mapping: |
|
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
|
- unarchive:
|
|
format: "json_array"
|
|
- mapping: |
|
|
if batch_index() == 0 {
|
|
meta topic = "logins"
|
|
root = this
|
|
} else {
|
|
meta topic = "transactions"
|
|
root.user_id = this.user_id
|
|
root.email = this.email
|
|
root.index = batch_index() - 1
|
|
root.product_url = fake("url")
|
|
root.price = fake("amount_with_currency")
|
|
root.timestamp = now()
|
|
}
|
|
output:
|
|
kafka_franz:
|
|
seed_brokers: [ "redpanda-0:9092" ]
|
|
topic: $${! metadata("topic") }
|
|
sasl:
|
|
- mechanism: SCRAM-SHA-256
|
|
password: secretpassword
|
|
username: superuser
|
|
####################
|
|
# rpk container to create the edu-filtered-domains topic #
|
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
|
####################
|
|
createtopic:
|
|
command:
|
|
- topic
|
|
- create
|
|
- edu-filtered-domains
|
|
- -X user=superuser
|
|
- -X pass=secretpassword
|
|
- -X brokers=redpanda-0:9092
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
networks:
|
|
- redpanda_network
|
|
depends_on:
|
|
redpanda-0:
|
|
condition: service_healthy
|
|
####################
|
|
# rpk container to register the schema #
|
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
|
####################
|
|
registerschema:
|
|
command:
|
|
- registry
|
|
- schema
|
|
- create
|
|
- transactions
|
|
- --schema
|
|
- /etc/redpanda/transactions-schema.json
|
|
- -X user=superuser
|
|
- -X pass=secretpassword
|
|
- -X registry.hosts=redpanda-0:8081
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
# Mount the local directory that contains your schema to the container.
|
|
volumes:
|
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
|
networks:
|
|
- redpanda_network
|
|
depends_on:
|
|
redpanda-0:
|
|
condition: service_healthy
|
|
####################
|
|
# rpk container to deploy a consumer group #
|
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
|
####################
|
|
consumergroup:
|
|
command:
|
|
- topic
|
|
- consume
|
|
- transactions
|
|
- --group
|
|
- transactions-consumer
|
|
- -X user=superuser
|
|
- -X pass=secretpassword
|
|
- -X brokers=redpanda-0:9092
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
networks:
|
|
- redpanda_network
|
|
depends_on:
|
|
createtopic:
|
|
condition: service_completed_successfully
|
|
deploytransform:
|
|
condition: service_completed_successfully
|
|
####################
|
|
# rpk container to deploy the pre-built data transform #
|
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
|
####################
|
|
deploytransform:
|
|
command:
|
|
- transform
|
|
- deploy
|
|
- --file=/etc/redpanda/regex.wasm
|
|
- --name=regex
|
|
- --input-topic=logins
|
|
- --output-topic=edu-filtered-domains
|
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
|
- --var=MATCH_VALUE=true
|
|
- -X user=superuser
|
|
- -X pass=secretpassword
|
|
- -X admin.hosts=redpanda-0:9644
|
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
|
volumes:
|
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
|
networks:
|
|
- redpanda_network
|
|
depends_on:
|
|
createtopic:
|
|
condition: service_completed_successfully
|
|
####################
|
|
# MinIO for Tiered Storage #
|
|
# See https://min.io/
|
|
#
|
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
|
# It is not supported for production deployments of Redpanda.
|
|
#
|
|
# For production environments, use one of the supported object storage providers:
|
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
|
####################
|
|
minio:
|
|
container_name: minio
|
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
|
command: server --console-address ":9001" /data
|
|
ports:
|
|
- 9000:9000
|
|
- 9001:9001
|
|
environment:
|
|
MINIO_ROOT_USER: minio
|
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
|
MINIO_SERVER_URL: "http://minio:9000"
|
|
MINIO_REGION_NAME: local
|
|
MINIO_DOMAIN: minio
|
|
volumes:
|
|
- minio:/data
|
|
networks:
|
|
redpanda_network:
|
|
aliases:
|
|
- redpanda.minio
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
mc:
|
|
depends_on:
|
|
minio:
|
|
condition: service_healthy
|
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
|
container_name: mc
|
|
networks:
|
|
- redpanda_network
|
|
environment:
|
|
- AWS_ACCESS_KEY_ID=minio
|
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
|
- AWS_REGION=local
|
|
entrypoint: >
|
|
/bin/sh -c "
|
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
|
/usr/bin/mc mb minio/redpanda;
|
|
/usr/bin/mc policy set public minio/redpanda;
|
|
tail -f /dev/null
|
|
"
|