Message Queue Example
Process messages from Kafka/RabbitMQ with filtering and transformation.
Overview
This example demonstrates:
- Message queue as event source
- Event filtering by topic/type
- Message transformation
- Dead letter queue handling
Kafka Consumer Example
go
package main
import (
"context"
"encoding/json"
"log"
"github.com/IBM/sarama"
"github.com/emad-elsaid/boolexpr"
fh "github.com/emad-elsaid/firehose"
"github.com/emad-elsaid/firehose/actions"
"github.com/emad-elsaid/firehose/condition"
)
// Event type
type OrderEvent struct {
OrderID string
Amount float64
Status string
}
func (o OrderEvent) Get(key string) (any, error) {
switch key {
case "amount":
return o.Amount, nil
case "status":
return o.Status, nil
default:
return nil, fmt.Errorf("unknown: %s", key)
}
}
// Kafka Source
type KafkaSource struct {
Brokers []string
Topic string
GroupID string
}
func (k KafkaSource) Start(
ctx context.Context,
cb fh.Callback[OrderEvent],
) (<-chan struct{}, error) {
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
consumer, err := sarama.NewConsumerGroup(k.Brokers, k.GroupID, config)
if err != nil {
return nil, err
}
handler := &consumerHandler{callback: cb}
go func() {
defer consumer.Close()
for {
if err := consumer.Consume(ctx, []string{k.Topic}, handler); err != nil {
log.Printf("Kafka error: %v", err)
}
if ctx.Err() != nil {
return
}
}
}()
return ctx.Done(), nil
}
type consumerHandler struct {
callback fh.Callback[OrderEvent]
}
func (h *consumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil }
func (h *consumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil }
func (h *consumerHandler) ConsumeClaim(
session sarama.ConsumerGroupSession,
claim sarama.ConsumerGroupClaim,
) error {
for message := range claim.Messages() {
var event OrderEvent
if err := json.Unmarshal(message.Value, &event); err != nil {
log.Printf("Parse error: %v", err)
continue
}
h.callback(session.Context(), event, func(err error) {
if err == nil {
session.MarkMessage(message, "")
} else {
log.Printf("Processing error: %v", err)
}
})
}
return nil
}
// Actions
type ProcessHighValueOrder struct{}
func (p ProcessHighValueOrder) Process(
ctx context.Context,
order OrderEvent,
_ boolexpr.Symbols,
) (OrderEvent, error) {
log.Printf("Processing high-value order: %s", order.OrderID)
return order, nil
}
// Destinations
type DatabaseWriter struct{}
func (d DatabaseWriter) Send(ctx context.Context, order OrderEvent) error {
log.Printf("Saved to database: %s", order.OrderID)
return nil
}
type DeadLetterQueue struct{}
func (d DeadLetterQueue) Send(ctx context.Context, order OrderEvent) error {
log.Printf("Sent to DLQ: %s", order.OrderID)
return nil
}
func main() {
ctx := context.Background()
kafkaSource := &KafkaSource{
Brokers: []string{"localhost:9092"},
Topic: "orders",
GroupID: "order-processor",
}
var head fh.Rule
var err error
// High-value order processing
head, err = fh.Add(ctx, head, &fh.SQLRule[OrderEvent, OrderEvent]{
ID: "high_value",
Select: ProcessHighValueOrder{},
Into: DatabaseWriter{},
From: kafkaSource,
Where: condition.Cond[OrderEvent](`amount > 1000`),
})
if err != nil {
log.Fatal(err)
}
// Failed order handling
head, err = fh.Add(ctx, head, &fh.SQLRule[OrderEvent, OrderEvent]{
ID: "failed_orders",
Select: actions.Func[OrderEvent, OrderEvent](func(ctx context.Context, e OrderEvent, _ boolexpr.Symbols) (OrderEvent, error) { return e, nil }),
Into: DeadLetterQueue{},
From: kafkaSource,
Where: condition.Cond[OrderEvent](`status = "failed"`),
})
if err != nil {
log.Fatal(err)
}
doneChannels := fh.Start(ctx, head, nil)
for _, ch := range doneChannels {
<-ch
}
}Key Concepts
- Message acknowledgment based on processing success
- High-value order filtering with conditions
- Dead letter queue for failed messages
- Single consumer group with source fanout