diff --git a/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml b/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml new file mode 100644 index 00000000..605937b1 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-monitoring-schema + namespace: freeleaps-data-platform +data: + schema.json: | + { + "schemaName": "kafka_metrics", + "dimensionFieldSpecs": [ + { + "name": "topic", + "dataType": "STRING" + }, + { + "name": "partition", + "dataType": "INT" + } + ], + "metricFieldSpecs": [ + { + "name": "messages_in_per_sec", + "dataType": "DOUBLE" + }, + { + "name": "bytes_in_per_sec", + "dataType": "DOUBLE" + } + ], + "timeFieldSpec": { + "incomingGranularitySpec": { + "timeType": "MILLISECONDS", + "timeFormat": "EPOCH", + "name": "timestamp", + "dataType": "LONG" + } + } + } + table.json: | + { + "tableName": "business_metrics", + "tableType": "REALTIME", + "segmentsConfig": { + "timeColumnName": "timestamp", + "timeType": "MILLISECONDS", + "retentionTimeUnit": "DAYS", + "retentionTimeValue": "90" + }, + "streamConfigs": { + "streamType": "kafka", + "stream.kafka.consumer.type": "lowlevel", + "stream.kafka.topic.name": "business-metrics", + "stream.kafka.broker.list": "kafka-headless.freeleaps-data-platform.svc.cluster.local:9092", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder" + }, + "indexingConfig": { + "loadMode": "MMAP", + "aggregateMetrics": true + } + } + \ No newline at end of file