redpanda using docker-compose
This commit is contained in:
priyanshu111299
56
docker-compose/bootstrap.yml
Normal file
56
docker-compose/bootstrap.yml
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
403
docker-compose/docker-compose.yml
Normal file
403
docker-compose/docker-compose.yml
Normal file
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
77
docker-compose/generate-profiles.yaml
Normal file
77
docker-compose/generate-profiles.yaml
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# =================================================================
|
||||||
|
# This file defines initial cluster properties for a Redpanda cluster.
|
||||||
|
# Some of these settings are intended for quickstart development and evaluation
|
||||||
|
# and are not suitable for production environments.
|
||||||
|
#
|
||||||
|
# For more information on bootstrap files, see:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
||||||
|
# =================================================================
|
||||||
|
|
||||||
|
#
|
||||||
|
# Enable SASL authentication for the Kafka and Admin APIs.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
||||||
|
admin_api_require_auth: true
|
||||||
|
# At least one superuser is required to be able to create other SASL users
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
||||||
|
superusers:
|
||||||
|
- superuser
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
||||||
|
enable_sasl: true
|
||||||
|
# Allow topics to be created on first access.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
||||||
|
auto_create_topics_enabled: true
|
||||||
|
# Enable data transforms.
|
||||||
|
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
||||||
|
data_transforms_enabled: true
|
||||||
|
# Enable audit logging (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/audit-logging/
|
||||||
|
audit_enabled: true
|
||||||
|
# Enable Tiered Storage (enterprise feature).
|
||||||
|
# https://docs.redpanda.com/current/manage/tiered-storage/
|
||||||
|
cloud_storage_enabled: true
|
||||||
|
cloud_storage_region: local
|
||||||
|
cloud_storage_access_key: minio
|
||||||
|
cloud_storage_secret_key: redpandaTieredStorage7
|
||||||
|
cloud_storage_api_endpoint: minio
|
||||||
|
cloud_storage_api_endpoint_port: 9000
|
||||||
|
cloud_storage_disable_tls: true
|
||||||
|
cloud_storage_bucket: redpanda
|
||||||
|
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
||||||
|
cloud_storage_segment_max_upload_interval_sec: 60
|
||||||
|
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
||||||
|
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
||||||
|
partition_autobalancing_mode: continuous
|
||||||
|
# Enable Redpanda to collect consumer group metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
||||||
|
enable_consumer_group_metrics:
|
||||||
|
- "group"
|
||||||
|
- "partition"
|
||||||
|
- "consumer_lag"
|
||||||
|
# Lower the interval for the quickstart
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
||||||
|
consumer_group_lag_collection_interval_sec: 60
|
||||||
|
# Enable Redpanda to collect host metrics.
|
||||||
|
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
||||||
|
enable_host_metrics: true
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
name: redpanda-quickstart-multi-broker
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
driver: bridge
|
||||||
|
volumes:
|
||||||
|
redpanda-0: null
|
||||||
|
redpanda-1: null
|
||||||
|
redpanda-2: null
|
||||||
|
minio: null
|
||||||
|
services:
|
||||||
|
##################
|
||||||
|
# Redpanda Brokers #
|
||||||
|
##################
|
||||||
|
redpanda-0:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
||||||
|
# Address the broker advertises to clients that connect to the Kafka API.
|
||||||
|
# Use the internal addresses to connect to the Redpanda brokers
|
||||||
|
# from inside the same Docker network.
|
||||||
|
# Use the external addresses to connect to the Redpanda brokers
|
||||||
|
# from outside the Docker network.
|
||||||
|
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
||||||
|
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||||
|
# Redpanda brokers use the RPC API to communicate with each other internally.
|
||||||
|
- --rpc-addr redpanda-0:33145
|
||||||
|
- --advertise-rpc-addr redpanda-0:33145
|
||||||
|
# Mode dev-container uses well-known configuration properties for development in containers.
|
||||||
|
- --mode dev-container
|
||||||
|
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-0
|
||||||
|
# Sets the username and password of the bootstrap SCRAM superuser
|
||||||
|
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-0:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 18081:18081
|
||||||
|
- 18082:18082
|
||||||
|
- 19092:19092
|
||||||
|
- 19644:9644
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 10
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
redpanda-1:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
||||||
|
- --rpc-addr redpanda-1:33145
|
||||||
|
- --advertise-rpc-addr redpanda-1:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-1
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-1:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 28081:28081
|
||||||
|
- 28082:28082
|
||||||
|
- 29092:29092
|
||||||
|
- 29644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
redpanda-2:
|
||||||
|
command:
|
||||||
|
- redpanda
|
||||||
|
- start
|
||||||
|
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
||||||
|
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
||||||
|
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
||||||
|
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
||||||
|
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
||||||
|
- --rpc-addr redpanda-2:33145
|
||||||
|
- --advertise-rpc-addr redpanda-2:33145
|
||||||
|
- --mode dev-container
|
||||||
|
- --smp 1
|
||||||
|
- --default-log-level=info
|
||||||
|
- --seeds redpanda-0:33145
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
container_name: redpanda-2
|
||||||
|
environment:
|
||||||
|
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
||||||
|
volumes:
|
||||||
|
- redpanda-2:/var/lib/redpanda/data
|
||||||
|
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
ports:
|
||||||
|
- 38081:38081
|
||||||
|
- 38082:38082
|
||||||
|
- 39092:39092
|
||||||
|
- 39644:9644
|
||||||
|
depends_on:
|
||||||
|
- redpanda-0
|
||||||
|
- minio
|
||||||
|
####################
|
||||||
|
# Redpanda Console #
|
||||||
|
####################
|
||||||
|
console:
|
||||||
|
container_name: redpanda-console
|
||||||
|
image: docker.redpanda.com/redpandadata/console:v3.2.2
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
||||||
|
volumes:
|
||||||
|
- ./config:/tmp/config/
|
||||||
|
environment:
|
||||||
|
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
||||||
|
CONSOLE_CONFIG_FILE: |
|
||||||
|
# Configure a connection to the Redpanda cluster
|
||||||
|
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
||||||
|
kafka:
|
||||||
|
brokers: ["redpanda-0:9092"]
|
||||||
|
sasl:
|
||||||
|
enabled: true
|
||||||
|
impersonateUser: true
|
||||||
|
schemaRegistry:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
||||||
|
authentication:
|
||||||
|
impersonateUser: true
|
||||||
|
redpanda:
|
||||||
|
adminApi:
|
||||||
|
enabled: true
|
||||||
|
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
||||||
|
authentication:
|
||||||
|
basic:
|
||||||
|
username: superuser
|
||||||
|
password: secretpassword
|
||||||
|
impersonateUser: false
|
||||||
|
console:
|
||||||
|
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
||||||
|
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
||||||
|
topicDocumentation:
|
||||||
|
enabled: true
|
||||||
|
git:
|
||||||
|
enabled: true
|
||||||
|
repository:
|
||||||
|
url: https://github.com/redpanda-data/docs
|
||||||
|
branch: main
|
||||||
|
baseDirectory: tests/docker-compose
|
||||||
|
authentication:
|
||||||
|
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
||||||
|
basic:
|
||||||
|
enabled: true
|
||||||
|
authorization:
|
||||||
|
roleBindings:
|
||||||
|
- roleName: admin
|
||||||
|
users:
|
||||||
|
- loginType: basic
|
||||||
|
name: superuser
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
registerschema:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# Redpanda Connect #
|
||||||
|
####################
|
||||||
|
connect:
|
||||||
|
container_name: redpanda-connect
|
||||||
|
image: docker.redpanda.com/redpandadata/connect
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
entrypoint: /bin/sh
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
||||||
|
environment:
|
||||||
|
# This Redpanda Connect configuration creates fake data,
|
||||||
|
# processes it, and writes the output to a set of topics.
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# - Uses Redpanda Connect's generate input to generate fake data.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
# Pipeline:
|
||||||
|
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
||||||
|
# topic, and a random number (1-3) of messages to 'transaction' topic
|
||||||
|
# - Unarchive processor to parse the JSON array and extract each
|
||||||
|
# element into its own message.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
# Output:
|
||||||
|
# - kafka_franz output to write the messages to the Redpanda brokers.
|
||||||
|
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
CONNECT_CFG_FILE: |
|
||||||
|
input:
|
||||||
|
generate:
|
||||||
|
interval: 1s
|
||||||
|
mapping: |
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
root.user_id = counter()
|
||||||
|
root.name = $$first_name + " " + $$last_name
|
||||||
|
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
root.ip = fake("ipv4")
|
||||||
|
root.login_time = now()
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
||||||
|
- unarchive:
|
||||||
|
format: "json_array"
|
||||||
|
- mapping: |
|
||||||
|
if batch_index() == 0 {
|
||||||
|
meta topic = "logins"
|
||||||
|
root = this
|
||||||
|
} else {
|
||||||
|
meta topic = "transactions"
|
||||||
|
root.user_id = this.user_id
|
||||||
|
root.email = this.email
|
||||||
|
root.index = batch_index() - 1
|
||||||
|
root.product_url = fake("url")
|
||||||
|
root.price = fake("amount_with_currency")
|
||||||
|
root.timestamp = now()
|
||||||
|
}
|
||||||
|
output:
|
||||||
|
kafka_franz:
|
||||||
|
seed_brokers: [ "redpanda-0:9092" ]
|
||||||
|
topic: $${! metadata("topic") }
|
||||||
|
sasl:
|
||||||
|
- mechanism: SCRAM-SHA-256
|
||||||
|
password: secretpassword
|
||||||
|
username: superuser
|
||||||
|
####################
|
||||||
|
# rpk container to create the edu-filtered-domains topic #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
||||||
|
####################
|
||||||
|
createtopic:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- create
|
||||||
|
- edu-filtered-domains
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to register the schema #
|
||||||
|
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
||||||
|
####################
|
||||||
|
registerschema:
|
||||||
|
command:
|
||||||
|
- registry
|
||||||
|
- schema
|
||||||
|
- create
|
||||||
|
- transactions
|
||||||
|
- --schema
|
||||||
|
- /etc/redpanda/transactions-schema.json
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X registry.hosts=redpanda-0:8081
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
# Mount the local directory that contains your schema to the container.
|
||||||
|
volumes:
|
||||||
|
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
redpanda-0:
|
||||||
|
condition: service_healthy
|
||||||
|
####################
|
||||||
|
# rpk container to deploy a consumer group #
|
||||||
|
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
||||||
|
####################
|
||||||
|
consumergroup:
|
||||||
|
command:
|
||||||
|
- topic
|
||||||
|
- consume
|
||||||
|
- transactions
|
||||||
|
- --group
|
||||||
|
- transactions-consumer
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X brokers=redpanda-0:9092
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploytransform:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# rpk container to deploy the pre-built data transform #
|
||||||
|
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
||||||
|
####################
|
||||||
|
deploytransform:
|
||||||
|
command:
|
||||||
|
- transform
|
||||||
|
- deploy
|
||||||
|
- --file=/etc/redpanda/regex.wasm
|
||||||
|
- --name=regex
|
||||||
|
- --input-topic=logins
|
||||||
|
- --output-topic=edu-filtered-domains
|
||||||
|
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
||||||
|
- --var=MATCH_VALUE=true
|
||||||
|
- -X user=superuser
|
||||||
|
- -X pass=secretpassword
|
||||||
|
- -X admin.hosts=redpanda-0:9644
|
||||||
|
image: docker.redpanda.com/redpandadata/redpanda:v25.2.7
|
||||||
|
volumes:
|
||||||
|
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
depends_on:
|
||||||
|
createtopic:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
####################
|
||||||
|
# MinIO for Tiered Storage #
|
||||||
|
# See https://min.io/
|
||||||
|
#
|
||||||
|
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
||||||
|
# It is not supported for production deployments of Redpanda.
|
||||||
|
#
|
||||||
|
# For production environments, use one of the supported object storage providers:
|
||||||
|
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
||||||
|
####################
|
||||||
|
minio:
|
||||||
|
container_name: minio
|
||||||
|
image: minio/minio:RELEASE.2025-05-24T17-08-30Z
|
||||||
|
command: server --console-address ":9001" /data
|
||||||
|
ports:
|
||||||
|
- 9000:9000
|
||||||
|
- 9001:9001
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minio
|
||||||
|
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
||||||
|
MINIO_SERVER_URL: "http://minio:9000"
|
||||||
|
MINIO_REGION_NAME: local
|
||||||
|
MINIO_DOMAIN: minio
|
||||||
|
volumes:
|
||||||
|
- minio:/data
|
||||||
|
networks:
|
||||||
|
redpanda_network:
|
||||||
|
aliases:
|
||||||
|
- redpanda.minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mc:
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
image: minio/mc:RELEASE.2025-05-21T01-59-54Z
|
||||||
|
container_name: mc
|
||||||
|
networks:
|
||||||
|
- redpanda_network
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=minio
|
||||||
|
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
||||||
|
- AWS_REGION=local
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
until (/usr/bin/mc alias set minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
||||||
|
/usr/bin/mc mb minio/redpanda;
|
||||||
|
/usr/bin/mc policy set public minio/redpanda;
|
||||||
|
tail -f /dev/null
|
||||||
|
"
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
input:
|
||||||
|
# Use the 'generate' input
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
||||||
|
generate:
|
||||||
|
# The interval at which new records are generated.
|
||||||
|
interval: 1s
|
||||||
|
# The mapping section defines how each generated record is structured.
|
||||||
|
# The language used here is called Bloblang.
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
||||||
|
mapping: |
|
||||||
|
# Generate a fake first name using the 'first_name' faker function.
|
||||||
|
let first_name = fake("first_name")
|
||||||
|
|
||||||
|
# Generate a fake last name using the 'last_name' faker function.
|
||||||
|
let last_name = fake("last_name")
|
||||||
|
|
||||||
|
# Define possible subscription levels for users.
|
||||||
|
let subscription_levels = ["Free", "Basic", "Premium"]
|
||||||
|
|
||||||
|
# Define possible notification channels for user preferences.
|
||||||
|
let notifications = ["email", "sms", "push" ]
|
||||||
|
|
||||||
|
# Define supported languages for user preferences.
|
||||||
|
let languages = ["en", "es", "fr", "de", "zh", "jp"]
|
||||||
|
|
||||||
|
# Assign a unique user ID using a UUID digit generator.
|
||||||
|
root.user_id = fake("uuid_digit")
|
||||||
|
|
||||||
|
# Assign the generated first name to the 'first_name' field.
|
||||||
|
root.first_name = $first_name
|
||||||
|
|
||||||
|
# Assign the generated last name to the 'last_name' field.
|
||||||
|
root.last_name = $last_name
|
||||||
|
|
||||||
|
# Construct the user's email by combining the first initial, last name, and a fake domain name.
|
||||||
|
# The email is converted to lowercase for consistency.
|
||||||
|
root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
|
||||||
|
|
||||||
|
# Assign a fake registration date using the 'date' faker function.
|
||||||
|
root.registration_date = fake("date")
|
||||||
|
|
||||||
|
# Assign the current timestamp as the last login time.
|
||||||
|
root.last_login = now()
|
||||||
|
|
||||||
|
# Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
|
||||||
|
root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
|
||||||
|
|
||||||
|
# Randomly assign a language preference by selecting an index from the 'languages' array.
|
||||||
|
root.preferences.language = $languages.index(random_int(min: 0, max: 5))
|
||||||
|
|
||||||
|
# Randomly assign a notification preference by selecting an index from the 'notifications' array.
|
||||||
|
root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
|
||||||
|
pipeline:
|
||||||
|
processors:
|
||||||
|
- mapping: |
|
||||||
|
# Set the target topic for the generated records to 'profiles'.
|
||||||
|
meta topic = "profiles"
|
||||||
|
|
||||||
|
# Assign the entire record (root) to be sent to the specified topic.
|
||||||
|
root = this
|
||||||
|
output:
|
||||||
|
# Use the 'kafka_franz' output to send the result back to Redpanda
|
||||||
|
# https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
||||||
|
kafka_franz:
|
||||||
|
# Define the list of seed brokers for the Kafka cluster.
|
||||||
|
seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
|
||||||
|
# Dynamically assign the topic based on the metadata specified in the processors.
|
||||||
|
# In this case, it resolves to the 'profiles' topic.
|
||||||
|
topic: ${! metadata("topic") }
|
||||||
|
# Configure SASL authentication to securely connect to the Kafka brokers.
|
||||||
|
sasl:
|
||||||
|
- # Specify the SASL mechanism to use for authentication.
|
||||||
|
mechanism: SCRAM-SHA-256
|
||||||
|
# The password for the SASL authentication.
|
||||||
|
password: secretpassword
|
||||||
|
# The username for the SASL authentication.
|
||||||
|
username: superuser
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Transactions",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"email": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "email",
|
||||||
|
"description": "The email address of the user involved in the transaction."
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric index associated with the transaction."
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^XXX \\d+\\.\\d{6}$",
|
||||||
|
"description": "A string representing the price of the product, including a currency code followed by the amount."
|
||||||
|
},
|
||||||
|
"product_url": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "uri",
|
||||||
|
"description": "A URL that points to the product involved in the transaction."
|
||||||
|
},
|
||||||
|
"timestamp": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
|
||||||
|
},
|
||||||
|
"user_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "A numeric identifier for the user."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# Transactions Topic Documentation
|
||||||
|
|
||||||
|
This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
|
||||||
|
|
||||||
|
## Schema Overview
|
||||||
|
|
||||||
|
Each message in the `transactions` topic adheres to the following JSON schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "string",
|
||||||
|
"index": "integer",
|
||||||
|
"price": "string",
|
||||||
|
"product_url": "string",
|
||||||
|
"timestamp": "string",
|
||||||
|
"user_id": "integer"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **email**: The email address of the user involved in the transaction.
|
||||||
|
- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
|
||||||
|
- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
|
||||||
|
- **product_url**: A URL that points to the product involved in the transaction.
|
||||||
|
- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
|
||||||
|
- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
|
||||||
|
|
||||||
|
## Example message
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"email": "wzieme@ykczius.edu",
|
||||||
|
"index": 0,
|
||||||
|
"price": "XXX 5651308.100000",
|
||||||
|
"product_url": "http://yjomdta.top/DxvGsCn.php",
|
||||||
|
"timestamp": "2024-08-16T15:51:19.799474084Z",
|
||||||
|
"user_id": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
You can use the `transactions` topic for various purposes, including:
|
||||||
|
|
||||||
|
- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
|
||||||
|
- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
|
||||||
|
- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
= Modify the Wasm Transform in the Quickstart
|
||||||
|
|
||||||
|
This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
|
||||||
|
If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
|
||||||
|
|
||||||
|
However, if you want to customize the data transform logic, continue reading.
|
||||||
|
|
||||||
|
== Why customize the transform?
|
||||||
|
|
||||||
|
- **Custom filtering**: Filter by a different regex or apply multiple conditions.
|
||||||
|
- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
|
||||||
|
- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
|
||||||
|
|
||||||
|
== Prerequisites
|
||||||
|
|
||||||
|
You need the following:
|
||||||
|
|
||||||
|
- At least Go 1.20 installed.
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
go version
|
||||||
|
----
|
||||||
|
|
||||||
|
- The Redpanda CLI (`rpk`) installed.
|
||||||
|
|
||||||
|
- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
|
||||||
|
|
||||||
|
== Modify and deploy your transform
|
||||||
|
|
||||||
|
. Open link:transform.go[transform.go] and make your changes. For example:
|
||||||
|
+
|
||||||
|
--
|
||||||
|
- Change the regex logic to handle different use cases.
|
||||||
|
- Add environment variables to control new features.
|
||||||
|
- Extend the `doRegexFilter()` function to manipulate records.
|
||||||
|
--
|
||||||
|
|
||||||
|
. Compile your Go code into a `.wasm` file:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk transform build
|
||||||
|
----
|
||||||
|
+
|
||||||
|
This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
|
||||||
|
|
||||||
|
. Deploy the new transform.
|
||||||
|
+
|
||||||
|
If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
|
||||||
|
+
|
||||||
|
Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
|
||||||
|
|
||||||
|
. Produce messages into the input topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
|
||||||
|
----
|
||||||
|
|
||||||
|
. Consume from the output topic. For example:
|
||||||
|
+
|
||||||
|
[source,bash]
|
||||||
|
----
|
||||||
|
rpk topic consume edu-filtered-domains --num 1
|
||||||
|
----
|
||||||
|
|
||||||
|
== Suggested reading
|
||||||
|
|
||||||
|
- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
|
||||||
|
- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
|
||||||
|
- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
|
||||||
|
- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module regex
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
|
||||||
|
github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=
|
||||||
Binary file not shown.
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
// This data transform filters records based on a customizable regex pattern.
|
||||||
|
// If a record's key or value
|
||||||
|
// (determined by an environment variable) matches the specified regex,
|
||||||
|
// the record is forwarded to the output.
|
||||||
|
// Otherwise, it is dropped.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// 1. Provide the following environment variables in your Docker or configuration setup:
|
||||||
|
// - PATTERN : (required) a regular expression that determines what you want to match.
|
||||||
|
// - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
|
||||||
|
// the record key is checked. Default is false.
|
||||||
|
//
|
||||||
|
// Example environment variables:
|
||||||
|
// PATTERN=".*\\.edu$"
|
||||||
|
// MATCH_VALUE="true"
|
||||||
|
//
|
||||||
|
// Logs:
|
||||||
|
// This transform logs information about each record and whether it matched.
|
||||||
|
// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
|
||||||
|
//
|
||||||
|
// Build instructions:
|
||||||
|
// go mod tidy
|
||||||
|
// rpk transform build
|
||||||
|
//
|
||||||
|
// For more details on building transforms with the Redpanda SDK, see:
|
||||||
|
// https://docs.redpanda.com/current/develop/data-transforms
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
re *regexp.Regexp
|
||||||
|
checkValue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
func isTrueVar(v string) bool {
|
||||||
|
switch strings.ToLower(v) {
|
||||||
|
case "yes", "ok", "1", "true":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The main() function runs only once at startup. It performs all initialization steps:
|
||||||
|
// - Reads and compiles the regex pattern.
|
||||||
|
// - Determines whether to match on the key or value.
|
||||||
|
// - Registers the doRegexFilter() function to process records.
|
||||||
|
func main() {
|
||||||
|
// Set logging preferences, including timestamp and UTC time.
|
||||||
|
log.SetPrefix("[regex-transform] ")
|
||||||
|
log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
|
||||||
|
|
||||||
|
// Start logging the transformation process
|
||||||
|
log.Println("Starting transform...")
|
||||||
|
|
||||||
|
// Read the PATTERN environment variable to get the regex pattern.
|
||||||
|
pattern, ok := os.LookupEnv("PATTERN")
|
||||||
|
if !ok {
|
||||||
|
log.Fatal("Missing PATTERN environment variable")
|
||||||
|
}
|
||||||
|
// Log the regex pattern being used.
|
||||||
|
log.Printf("Using PATTERN: %q\n", pattern)
|
||||||
|
// Compile the regex pattern for later use.
|
||||||
|
re = regexp.MustCompile(pattern)
|
||||||
|
|
||||||
|
// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
|
||||||
|
mk, ok := os.LookupEnv("MATCH_VALUE")
|
||||||
|
checkValue = ok && isTrueVar(mk)
|
||||||
|
log.Printf("MATCH_VALUE set to: %t\n", checkValue)
|
||||||
|
|
||||||
|
log.Println("Initialization complete, waiting for records...")
|
||||||
|
|
||||||
|
// Listen for records to be written, calling doRegexFilter() for each record.
|
||||||
|
transform.OnRecordWritten(doRegexFilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The doRegexFilter() function executes each time a new record is written.
|
||||||
|
// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
|
||||||
|
// If it matches, the record is forwarded, if not, it's dropped.
|
||||||
|
func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
|
||||||
|
// This stores the data to be checked (either the key or value).
|
||||||
|
var dataToCheck []byte
|
||||||
|
|
||||||
|
// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
|
||||||
|
if checkValue {
|
||||||
|
// Use the value of the record if MATCH_VALUE is true.
|
||||||
|
dataToCheck = e.Record().Value
|
||||||
|
log.Printf("Checking record value: %s\n", string(dataToCheck))
|
||||||
|
} else {
|
||||||
|
// Use the key of the record if MATCH_VALUE is false.
|
||||||
|
dataToCheck = e.Record().Key
|
||||||
|
log.Printf("Checking record key: %s\n", string(dataToCheck))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no key or value to check, log and skip the record.
|
||||||
|
if dataToCheck == nil {
|
||||||
|
log.Println("Record has no key/value to check, skipping.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the data matches the regex pattern.
|
||||||
|
pass := re.Match(dataToCheck)
|
||||||
|
if pass {
|
||||||
|
// If the record matches the pattern, log and write the record to the output topic.
|
||||||
|
log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
return w.Write(e.Record())
|
||||||
|
} else {
|
||||||
|
// If the record does not match the pattern, log and drop the record.
|
||||||
|
log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
|
||||||
|
// Do not write the record if it doesn't match the pattern.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Transform metadata used by the rpk transform build command.
|
||||||
|
# This metadata file tells rpk:
|
||||||
|
# 1) The transform’s display name, which also becomes the base for the .wasm file name.
|
||||||
|
# 2) A brief description of what it does.
|
||||||
|
# 3) Defaults for environment variables.
|
||||||
|
# 4) Input and output topics (if you want to define them here rather than in the deploy command).
|
||||||
|
|
||||||
|
# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
|
||||||
|
name: regex
|
||||||
|
|
||||||
|
description: |
|
||||||
|
Filters the input topic to records that only match a regular expression.
|
||||||
|
|
||||||
|
Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
|
||||||
|
See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
- PATTERN: The regular expression that will match against records (required).
|
||||||
|
- MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
|
||||||
|
|
||||||
|
# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
input-topic: ""
|
||||||
|
|
||||||
|
# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
|
||||||
|
output-topic: ""
|
||||||
|
|
||||||
|
# Indicates the specific TinyGo environment used to compile your transform.
|
||||||
|
language: tinygo-no-goroutines
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The PATTERN variable must be provided at deploy time.
|
||||||
|
# Example: --var=PATTERN=".*@example.com"
|
||||||
|
PATTERN: '<required>'
|
||||||
24
docker-compose/rpk-profile.yaml
Normal file
24
docker-compose/rpk-profile.yaml
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
|
||||||
|
|
||||||
|
# Configuration for connecting to the Kafka API of the Redpanda cluster.
|
||||||
|
kafka_api:
|
||||||
|
# SASL (Simple Authentication and Security Layer) settings for authentication.
|
||||||
|
sasl:
|
||||||
|
user: superuser # The username used for authentication
|
||||||
|
password: secretpassword # The password associated with the username
|
||||||
|
mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
|
||||||
|
# List of Kafka brokers in the Redpanda cluster.
|
||||||
|
# These brokers ensure high availability and fault tolerance for Kafka-based communication.
|
||||||
|
brokers:
|
||||||
|
- 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
|
||||||
|
- 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
|
||||||
|
- 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
|
||||||
|
|
||||||
|
# Configuration for connecting to the Redpanda Admin API.
|
||||||
|
# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
|
||||||
|
admin_api:
|
||||||
|
# List of Admin API endpoints for managing the cluster.
|
||||||
|
addresses:
|
||||||
|
- 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
|
||||||
|
- 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
|
||||||
|
- 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user