From 8930ff1c88a3be55e2c14c979238a347dab33de4 Mon Sep 17 00:00:00 2001
From: Ankit Nayan <ankit@signoz.io>
Date: Tue, 19 Jan 2021 18:30:17 +0530
Subject: [PATCH] added druid environments for small instance

---
 deploy/docker/docker-compose.yaml             | 10 ++--
 deploy/docker/environment_small/broker        | 53 +++++++++++++++++++
 deploy/docker/environment_small/coordinator   | 52 ++++++++++++++++++
 deploy/docker/environment_small/historical    | 53 +++++++++++++++++++
 .../middlemanager}                            | 11 ++--
 deploy/docker/environment_small/router        | 52 ++++++++++++++++++
 6 files changed, 221 insertions(+), 10 deletions(-)
 create mode 100644 deploy/docker/environment_small/broker
 create mode 100644 deploy/docker/environment_small/coordinator
 create mode 100644 deploy/docker/environment_small/historical
 rename deploy/docker/{environment => environment_small/middlemanager} (85%)
 create mode 100644 deploy/docker/environment_small/router

diff --git a/deploy/docker/docker-compose.yaml b/deploy/docker/docker-compose.yaml
index 19a226ff..2b2323d1 100644
--- a/deploy/docker/docker-compose.yaml
+++ b/deploy/docker/docker-compose.yaml
@@ -68,7 +68,7 @@ services:
     command:
       - coordinator
     env_file:
-      - environment
+      - environment_small/coordinator
 
   broker:
     image: apache/druid:0.20.0
@@ -84,7 +84,7 @@ services:
     command:
       - broker
     env_file:
-      - environment
+      - environment_small/broker
 
   historical:
     image: apache/druid:0.20.0
@@ -101,7 +101,7 @@ services:
     command:
       - historical
     env_file:
-      - environment
+      - environment_small/historical
 
   middlemanager:
     image: apache/druid:0.20.0
@@ -118,7 +118,7 @@ services:
     command:
       - middleManager
     env_file:
-      - environment
+      - environment_small/middlemanager
 
   router:
     image: apache/druid:0.20.0
@@ -134,7 +134,7 @@ services:
     command:
       - router
     env_file:
-      - environment
+      - environment_small/router
 
   flatten-processor:
     image: signoz/flattener-processor:0.1.1
diff --git a/deploy/docker/environment_small/broker b/deploy/docker/environment_small/broker
new file mode 100644
index 00000000..2b76b2c6
--- /dev/null
+++ b/deploy/docker/environment_small/broker
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=512m
+DRUID_XMS=512m
+DRUID_MAXNEWSIZE=256m
+DRUID_NEWSIZE=256m
+DRUID_MAXDIRECTMEMORYSIZE=768m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=768m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+druid_processing_buffer_sizeBytes=100MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=1
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
\ No newline at end of file
diff --git a/deploy/docker/environment_small/coordinator b/deploy/docker/environment_small/coordinator
new file mode 100644
index 00000000..25ec3847
--- /dev/null
+++ b/deploy/docker/environment_small/coordinator
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=64m
+DRUID_XMS=64m
+DRUID_MAXNEWSIZE=256m
+DRUID_NEWSIZE=256m
+DRUID_MAXDIRECTMEMORYSIZE=400m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xms64m", "-Xmx64m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=1
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
\ No newline at end of file
diff --git a/deploy/docker/environment_small/historical b/deploy/docker/environment_small/historical
new file mode 100644
index 00000000..5560087a
--- /dev/null
+++ b/deploy/docker/environment_small/historical
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=512m
+DRUID_XMS=512m
+DRUID_MAXNEWSIZE=256m
+DRUID_NEWSIZE=256m
+DRUID_MAXDIRECTMEMORYSIZE=1280m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=1280m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+druid_processing_buffer_sizeBytes=200MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
\ No newline at end of file
diff --git a/deploy/docker/environment b/deploy/docker/environment_small/middlemanager
similarity index 85%
rename from deploy/docker/environment
rename to deploy/docker/environment_small/middlemanager
index 4ccc70da..18bf3c41 100644
--- a/deploy/docker/environment
+++ b/deploy/docker/environment_small/middlemanager
@@ -20,9 +20,9 @@
 # Java tuning
 DRUID_XMX=1g
 DRUID_XMS=1g
-DRUID_MAXNEWSIZE=250m
-DRUID_NEWSIZE=250m
-DRUID_MAXDIRECTMEMORYSIZE=3072m
+DRUID_MAXNEWSIZE=256m
+DRUID_NEWSIZE=256m
+DRUID_MAXDIRECTMEMORYSIZE=2g
 
 druid_emitter_logging_logLevel=debug
 
@@ -38,8 +38,9 @@ druid_metadata_storage_connector_password=FoolishPassword
 
 druid_coordinator_balancer_strategy=cachingCost
 
-druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
+druid_indexer_runner_javaOptsArray=["-server", "-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=2g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+druid_processing_buffer_sizeBytes=200MiB
 
 druid_storage_type=local
 druid_storage_storageDirectory=/opt/data/segments
diff --git a/deploy/docker/environment_small/router b/deploy/docker/environment_small/router
new file mode 100644
index 00000000..56ee35cd
--- /dev/null
+++ b/deploy/docker/environment_small/router
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=128m
+DRUID_XMS=128m
+DRUID_MAXNEWSIZE=256m
+DRUID_NEWSIZE=256m
+DRUID_MAXDIRECTMEMORYSIZE=128m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xms128m", "-Xmx128m", "-XX:MaxDirectMemorySize=128m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=1
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
\ No newline at end of file
-- 
GitLab