Submit
Favicon of Druid

Druid

Execute OLAP queries in milliseconds on trillions of rows. Build highly concurrent applications with native integration for Kafka and Kinesis.

Visit DruidConfiguration

A high-performance, real-time analytics database designed for sub-second queries on streaming and batch data at scale. Execute OLAP queries in milliseconds on high-cardinality and high-dimensional data sets with billions to trillions of rows. Build real-time analytics applications that support hundreds to thousands of queries per second with consistent performance. It unlocks streaming data potential through native integration with Apache Kafka and Amazon Kinesis, supporting query-on-arrival at millions of events per second.

Key capabilities include:

  • Interactive Query Engine: Utilizes scatter/gather for high-speed queries with data preloaded into memory or local storage to avoid network latency.
  • Optimized Data Format: Ingested data is automatically columnarized, time-indexed, dictionary-encoded, bitmap-indexed, and type-aware compressed.
  • True Stream Ingestion: A connector-free integration with streaming platforms enables query-on-arrival, high scalability, and low latency.
  • Elastic Architecture: Loosely coupled components for ingestion, queries, and orchestration combined with a deep storage layer enable quick scaling.
  • SQL Support: Developers and analysts can use the familiar SQL API for end-to-end data operations across ingestion, transformation, and querying.
  • Schema Auto-Discovery: Automatically detects and defines column names and data types upon ingestion.

Directory Structure

druid
data
druid
deepstorage
indexing-logs
segment-cache
task-hadoop-tmp
postgres
zookeeper
data
datalog
.env
docker-compose.yml

docker-compose.yml

services:
  postgres:
    image: postgres:15
    container_name: druid-postgres
    volumes:
      - ./data/postgres:/var/lib/postgresql/data
    environment:
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_DB=${POSTGRES_DB}
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
      interval: 10s
      timeout: 5s
      retries: 5

  zookeeper:
    image: zookeeper:3.8
    container_name: druid-zookeeper
    volumes:
      - ./data/zookeeper/data:/data
      - ./data/zookeeper/datalog:/datalog
    restart: unless-stopped

  coordinator:
    image: apache/druid:latest
    container_name: druid-coordinator
    command:
      - coordinator
    volumes:
      - ./data/druid/deepstorage:/opt/druid/var/deepstorage
      - ./data/druid/indexing-logs:/opt/druid/var/indexing-logs
    depends_on:
      zookeeper: 
        condition: service_started
      postgres: 
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      - DRUID_XMX=1g
      - DRUID_XMS=1g
      - druid_metadata_storage_type=postgresql
      - druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/${POSTGRES_DB}
      - druid_metadata_storage_connector_user=${POSTGRES_USER}
      - druid_metadata_storage_connector_password=${POSTGRES_PASSWORD}
      - druid_zk_service_host=zookeeper
      - druid_zk_paths_base=/druid
      - druid_storage_type=local
      - druid_storage_storageDirectory=/opt/druid/var/deepstorage
      - druid_indexer_logs_type=file
      - druid_indexer_logs_directory=/opt/druid/var/indexing-logs
    restart: unless-stopped

  broker:
    image: apache/druid:latest
    container_name: druid-broker
    command:
      - broker
    depends_on:
      zookeeper: 
        condition: service_started
      coordinator: 
        condition: service_started
      postgres: 
        condition: service_healthy
    ports:
      - "8082:8082"
    environment:
      - DRUID_XMX=1g
      - DRUID_XMS=1g
      - druid_metadata_storage_type=postgresql
      - druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/${POSTGRES_DB}
      - druid_metadata_storage_connector_user=${POSTGRES_USER}
      - druid_metadata_storage_connector_password=${POSTGRES_PASSWORD}
      - druid_zk_service_host=zookeeper
      - druid_zk_paths_base=/druid
      - druid_sql_enable=true
    restart: unless-stopped

  historical:
    image: apache/druid:latest
    container_name: druid-historical
    command:
      - historical
    volumes:
      - ./data/druid/deepstorage:/opt/druid/var/deepstorage
      - ./data/druid/segment-cache:/opt/druid/var/druid/segment-cache
    depends_on:
      zookeeper: 
        condition: service_started
      coordinator: 
        condition: service_started
      postgres: 
        condition: service_healthy
    ports:
      - "8083:8083"
    environment:
      - DRUID_XMX=1g
      - DRUID_XMS=1g
      - druid_metadata_storage_type=postgresql
      - druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/${POSTGRES_DB}
      - druid_metadata_storage_connector_user=${POSTGRES_USER}
      - druid_metadata_storage_connector_password=${POSTGRES_PASSWORD}
      - druid_zk_service_host=zookeeper
      - druid_zk_paths_base=/druid
      - druid_storage_type=local
      - druid_storage_storageDirectory=/opt/druid/var/deepstorage
      - druid_segmentCache_locations=[{"path":"/opt/druid/var/druid/segment-cache","maxSize":10000000000}]
      - druid_server_maxSize=10000000000
    restart: unless-stopped

  middlemanager:
    image: apache/druid:latest
    container_name: druid-middlemanager
    command:
      - middleManager
    volumes:
      - ./data/druid/deepstorage:/opt/druid/var/deepstorage
      - ./data/druid/indexing-logs:/opt/druid/var/indexing-logs
      - ./data/druid/task-hadoop-tmp:/opt/druid/var/druid/task-hadoop-tmp
    depends_on:
      zookeeper: 
        condition: service_started
      coordinator: 
        condition: service_started
      postgres: 
        condition: service_healthy
    ports:
      - "8091:8091"
      - "8100-8105:8100-8105"
    environment:
      - DRUID_XMX=256m
      - DRUID_XMS=256m
      - druid_metadata_storage_type=postgresql
      - druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/${POSTGRES_DB}
      - druid_metadata_storage_connector_user=${POSTGRES_USER}
      - druid_metadata_storage_connector_password=${POSTGRES_PASSWORD}
      - druid_zk_service_host=zookeeper
      - druid_zk_paths_base=/druid
      - druid_storage_type=local
      - druid_storage_storageDirectory=/opt/druid/var/deepstorage
      - druid_indexer_logs_type=file
      - druid_indexer_logs_directory=/opt/druid/var/indexing-logs
      - druid_worker_capacity=3
      - druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
    restart: unless-stopped

  router:
    image: apache/druid:latest
    container_name: druid-router
    command:
      - router
    depends_on:
      zookeeper: 
        condition: service_started
      coordinator: 
        condition: service_started
    ports:
      - "8888:8888"
    environment:
      - DRUID_XMX=256m
      - DRUID_XMS=256m
      - druid_metadata_storage_type=postgresql
      - druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/${POSTGRES_DB}
      - druid_metadata_storage_connector_user=${POSTGRES_USER}
      - druid_metadata_storage_connector_password=${POSTGRES_PASSWORD}
      - druid_zk_service_host=zookeeper
      - druid_zk_paths_base=/druid
      - druid_router_managementProxy_enabled=true
    restart: unless-stopped

.env

POSTGRES_USER=druid
POSTGRES_PASSWORD=your_secure_password
POSTGRES_DB=druid
Categories:
Analytics

Tags:

analyticsapachebig-datadata-engineeringdatabasedistributed-systemsdruidhigh-performancejavakafkareal-timerescripttime-seriestypescript

Share:

Ad
Favicon

 

  
 

Similar to Druid

Favicon

 

  
  
Favicon

 

  
  
Favicon