sharmalynk/MLOPS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redpanda using docker-compose

Browse Source
This commit is contained in:
priyanshu111299
2025-10-06 19:52:00 +00:00
parent e003bc130a
commit ff812ab021
108 changed files with 7902 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

403
docker-compose/docker-compose.yml Normal file
View File

@@ -0,0 +1,403 @@
name: redpanda-quickstart-multi-broker
networks:
redpanda_network:
driver: bridge
volumes:
redpanda-0: null
redpanda-1: null
redpanda-2: null
minio: null
services:
##################
# Redpanda Brokers #
##################
redpanda-0:
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
# Address the broker advertises to clients that connect to the Kafka API.
# Use the internal addresses to connect to the Redpanda brokers
# from inside the same Docker network.
# Use the external addresses to connect to the Redpanda brokers
# from outside the Docker network.
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
# Address the broker advertises to clients that connect to the HTTP Proxy.
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
# Redpanda brokers use the RPC API to communicate with each other internally.
- --rpc-addr redpanda-0:33145
- --advertise-rpc-addr redpanda-0:33145
# Mode dev-container uses well-known configuration properties for development in containers.
- --mode dev-container
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
- --smp 1
- --default-log-level=info
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
container_name: redpanda-0
# Sets the username and password of the bootstrap SCRAM superuser
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
environment:
RP_BOOTSTRAP_USER: "superuser:secretpassword"
volumes:
- redpanda-0:/var/lib/redpanda/data
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
networks:
- redpanda_network
ports:
- 18081:18081
- 18082:18082
- 19092:19092
- 19644:9644
healthcheck:
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
interval: 10s
timeout: 15s
retries: 10
depends_on:
minio:
condition: service_healthy
redpanda-1:
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
- --rpc-addr redpanda-1:33145
- --advertise-rpc-addr redpanda-1:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
- --seeds redpanda-0:33145
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
container_name: redpanda-1
environment:
RP_BOOTSTRAP_USER: "superuser:secretpassword"
volumes:
- redpanda-1:/var/lib/redpanda/data
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
networks:
- redpanda_network
ports:
- 28081:28081
- 28082:28082
- 29092:29092
- 29644:9644
depends_on:
- redpanda-0
- minio
redpanda-2:
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
- --rpc-addr redpanda-2:33145
- --advertise-rpc-addr redpanda-2:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
- --seeds redpanda-0:33145
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
container_name: redpanda-2
environment:
RP_BOOTSTRAP_USER: "superuser:secretpassword"
volumes:
- redpanda-2:/var/lib/redpanda/data
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
networks:
- redpanda_network
ports:
- 38081:38081
- 38082:38082
- 39092:39092
- 39644:9644
depends_on:
- redpanda-0
- minio
####################
# Redpanda Console #
####################
console:
container_name: redpanda-console
image: docker.redpanda.com/redpandadata/console:v3.2.2
networks:
- redpanda_network
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
volumes:
- ./config:/tmp/config/
environment:
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
CONSOLE_CONFIG_FILE: |
# Configure a connection to the Redpanda cluster
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
kafka:
brokers: ["redpanda-0:9092"]
sasl:
enabled: true
impersonateUser: true
schemaRegistry:
enabled: true
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
authentication:
impersonateUser: true
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
authentication:
basic:
username: superuser
password: secretpassword
impersonateUser: false
console:
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
# See https://docs.redpanda.com/current/console/config/topic-documentation/
topicDocumentation:
enabled: true
git:
enabled: true
repository:
url: https://github.com/redpanda-data/docs
branch: main
baseDirectory: tests/docker-compose
authentication:
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
basic:
enabled: true
authorization:
roleBindings:
- roleName: admin
users:
- loginType: basic
name: superuser
ports:
- 8080:8080
depends_on:
redpanda-0:
condition: service_healthy
createtopic:
condition: service_completed_successfully
registerschema:
condition: service_completed_successfully
deploytransform:
condition: service_completed_successfully
####################
# Redpanda Connect #
####################
connect:
container_name: redpanda-connect
image: docker.redpanda.com/redpandadata/connect
networks:
- redpanda_network
entrypoint: /bin/sh
depends_on:
redpanda-0:
condition: service_healthy
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
environment:
# This Redpanda Connect configuration creates fake data,
# processes it, and writes the output to a set of topics.
#
# Input:
# - Uses Redpanda Connect's generate input to generate fake data.
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
# Pipeline:
# - Bloblang mapping to batch each input and map 1 message to 'logins'
# topic, and a random number (1-3) of messages to 'transaction' topic
# - Unarchive processor to parse the JSON array and extract each
# element into its own message.
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
# Output:
# - kafka_franz output to write the messages to the Redpanda brokers.
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
CONNECT_CFG_FILE: |
input:
generate:
interval: 1s
mapping: |
let first_name = fake("first_name")
let last_name = fake("last_name")
root.user_id = counter()
root.name = $$first_name + " " + $$last_name
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
root.ip = fake("ipv4")
root.login_time = now()
pipeline:
processors:
- mapping: |
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
- unarchive:
format: "json_array"
- mapping: |
if batch_index() == 0 {
meta topic = "logins"
root = this
} else {
meta topic = "transactions"
root.user_id = this.user_id
root.email = this.email
root.index = batch_index() - 1
root.product_url = fake("url")
root.price = fake("amount_with_currency")
root.timestamp = now()
}
output:
kafka_franz:
seed_brokers: [ "redpanda-0:9092" ]
topic: $${! metadata("topic") }
sasl:
- mechanism: SCRAM-SHA-256
password: secretpassword
username: superuser
####################
# rpk container to create the edu-filtered-domains topic #
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
####################
createtopic:
command:
- topic
- create
- edu-filtered-domains
- -X user=superuser
- -X pass=secretpassword
- -X brokers=redpanda-0:9092
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
networks:
- redpanda_network
depends_on:
redpanda-0:
condition: service_healthy
####################
# rpk container to register the schema #
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
####################
registerschema:
command:
- registry
- schema
- create
- transactions
- --schema
- /etc/redpanda/transactions-schema.json
- -X user=superuser
- -X pass=secretpassword
- -X registry.hosts=redpanda-0:8081
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
# Mount the local directory that contains your schema to the container.
volumes:
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
networks:
- redpanda_network
depends_on:
redpanda-0:
condition: service_healthy
####################
# rpk container to deploy a consumer group #
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
####################
consumergroup:
command:
- topic
- consume
- transactions
- --group
- transactions-consumer
- -X user=superuser
- -X pass=secretpassword
- -X brokers=redpanda-0:9092
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
networks:
- redpanda_network
depends_on:
createtopic:
condition: service_completed_successfully
deploytransform:
condition: service_completed_successfully
####################
# rpk container to deploy the pre-built data transform #
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
####################
deploytransform:
command:
- transform
- deploy
- --file=/etc/redpanda/regex.wasm
- --name=regex
- --input-topic=logins
- --output-topic=edu-filtered-domains
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
- --var=MATCH_VALUE=true
- -X user=superuser
- -X pass=secretpassword
- -X admin.hosts=redpanda-0:9644
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
volumes:
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
networks:
- redpanda_network
depends_on:
createtopic:
condition: service_completed_successfully
####################
# MinIO for Tiered Storage #
# See https://min.io/
#
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
# It is not supported for production deployments of Redpanda.
#
# For production environments, use one of the supported object storage providers:
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
####################
minio:
container_name: minio
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
command: server --console-address ":9001" /data
ports:
- 9000:9000
- 9001:9001
environment:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
MINIO_SERVER_URL: "http://minio:9000"
MINIO_REGION_NAME: local
MINIO_DOMAIN: minio
volumes:
- minio:/data
networks:
redpanda_network:
aliases:
- redpanda.minio
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
interval: 10s
timeout: 5s
retries: 3
mc:
depends_on:
minio:
condition: service_healthy
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
container_name: mc
networks:
- redpanda_network
environment:
- AWS_ACCESS_KEY_ID=minio
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
- AWS_REGION=local
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc mb minio/redpanda;
/usr/bin/mc policy set public minio/redpanda;
tail -f /dev/null
"