Merge pull request #24 from humio/grant/xmx-configurable

Grant/xmx configurable

defaults/main.yml

@@ -12,10 +12,13 @@ kafka_broker_id: 0
 kafka_broker_rack: dc1
 kafka_log_dirs: "/var/kafka/data"
 kafka_jmx_opts: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
-kafka_opts: "-Djava.rmi.server.useCodebaseOnly=true -Dcom.sun.jndi.ldap.object.trustURLCodebase=false -Dcom.sun.jndi.cosnaming.object.trustURLCodebase=false"
+kafka_opts: "-Djava.rmi.server.useCodebaseOnly=true -Dcom.sun.jndi.ldap.object.trustURLCodebase=false -Dcom.sun.jndi.cosnaming.object.trustURLCodebase=false -Dlog4j.configuration=file:/etc/kafka/log4j.properties"
 
 kafka_filebeat_fields: {}
 kafka_filebeat_tags: []
 
+kafka_xmx: "{{ ansible_memtotal_mb|int * 0.85|  round(0) | int  }}M"
+
+
 zookeeper_hosts:
 - ip: "{{ ansible_default_ipv4.address }}"

tasks/main.yml

@@ -69,6 +69,14 @@
     mode: 0644
   notify: Restart Kafka
 
+- name: Kafka log4j configuration
+  template:
+    src: "log4j.properties.j2"
+    dest: /etc/kafka/log4j.properties
+    mode: 0644
+  notify: Restart Kafka
+
+
 - name: Create Kafka SystemD unit configuration
   template:
     src: "kafka.service.j2"

templates/kafka.service.j2

@@ -7,7 +7,7 @@
   LimitNOFILE=800000
   Environment="LOG_DIR=/var/log/kafka"
   Environment="GC_LOG_ENABLED=true"
-  Environment="KAFKA_HEAP_OPTS=-Xms512M -Xmx4G"
+  Environment="KAFKA_HEAP_OPTS=-Xms512M -Xmx{{ kafka_xmx }}"
   Environment="NUMA_NODES={{ ansible_local.kafka_numanodes["0"].nodes }}"
   Environment="KAFKA_JMX_OPTS={{ kafka_jmx_opts }}"
   Environment="KAFKA_OPTS={{ kafka_opts }}"

templates/log4j.properties.j2

@@ -0,0 +1,84 @@
+
+
+
+# Unspecified loggers and loggers with additivity=true output to server.log and stdout
+# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
+log4j.rootLogger=INFO, stdout, kafkaAppender
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.kafkaAppender.MaxFileSize=50MB
+log4j.appender.kafkaAppender.MaxBackupIndex=10
+log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.stateChangeAppender.MaxFileSize=50MB
+log4j.appender.stateChangeAppender.MaxBackupIndex=10
+log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.requestAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.requestAppender.MaxFileSize=50MB
+log4j.appender.requestAppender.MaxBackupIndex=10
+log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.cleanerAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.cleanerAppender.MaxFileSize=50MB
+log4j.appender.cleanerAppender.MaxBackupIndex=10
+log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.controllerAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.controllerAppender.MaxFileSize=50MB
+log4j.appender.controllerAppender.MaxBackupIndex=10
+log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.authorizerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.authorizerAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.authorizerAppender.MaxFileSize=50MB
+log4j.appender.authorizerAppender.MaxBackupIndex=10
+log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Change the line below to adjust ZK client logging
+log4j.logger.org.apache.zookeeper=INFO
+
+# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
+log4j.logger.kafka=INFO
+log4j.logger.org.apache.kafka=INFO
+
+# Change to DEBUG or TRACE to enable request logging
+log4j.logger.kafka.request.logger=WARN, requestAppender
+log4j.additivity.kafka.request.logger=false
+
+# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
+# related to the handling of requests
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
+#log4j.additivity.kafka.server.KafkaApis=false
+log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
+log4j.additivity.kafka.network.RequestChannel$=false
+
+log4j.logger.kafka.controller=TRACE, controllerAppender
+log4j.additivity.kafka.controller=false
+
+log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
+log4j.additivity.kafka.log.LogCleaner=false
+
+log4j.logger.state.change.logger=INFO, stateChangeAppender
+log4j.additivity.state.change.logger=false
+
+# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
+log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
+log4j.additivity.kafka.authorizer.logger=false