WIP: Kafka source

.github/workflows/jekyll-gh-pages.yml

@@ -23,7 +23,7 @@ permissions:
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # Allow canceling of the in-progress deployment now.
 concurrency:
-  group: "pages"
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:

_data/link_aliases.yml

@@ -117,6 +117,12 @@ adm-src-program:
 adm-dest-program:
   aliases: [ "program() destination" ]
 
+adm-src-kafka:
+  aliases: [ "kafka() source" ]
+
+adm-dest-kafkac:
+  aliases: [ "kafka() destination" ]
+
 adm-src-mqtt:
   aliases: [ "mqtt() source" ]
 

_data/navigation.yml

@@ -165,6 +165,11 @@ admin-guide-nav:
     - title: "Jellyfin"
       url: /admin-guide/060_Sources/035_Jellyfin/README
       subnav:
+    - title: "kafka"
+      url: /admin-guide/060_Sources/038_Kafka/README
+      subnav:
+      - title: "Options of the kafka() source"
+        url: /admin-guide/060_Sources/038_Kafka/001_Kafka_options
     - title: "kubernetes"
       url: /admin-guide/060_Sources/040_Kubernetes/README
       subnav:
@@ -1159,7 +1164,7 @@ dev-guide-nav:
           subnav:
           - title: "file() Destination Driver"
             url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/file-destination-driver
-          - title: "file() Source Driver (DEPRECATED)"
+          - title: "file() Source Driver"
             url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/file-source-driver
           - title: "pipe() Destination Driver"
             url: /dev-guide/chapter_4/section_2/macos-testing-status/affile/pipe-destination-driver

_includes/doc/admin-guide/options/bootstrap-servers.md

@@ -0,0 +1,17 @@
+## bootstrap-servers()
+
+|  Type:   | string |
+|Default:  |  N/A   |
+|Mandatory:|  yes   |
+
+*Description:* Specifies the hostname or IP address of the Kafka server.
+When specifying an IP address, IPv4 (for example, 192.168.0.1) or IPv6
+(for example, \[::1\]) can be used as well. Use a colon (**:**) after
+the address to specify the port number of the server. When specifying
+multiple addresses, use a comma to separate the addresses, for example:
+
+``` config
+bootstrap-servers(
+    "127.0.0.1:2525,remote-server-hostname:6464"
+)
+```

_includes/doc/admin-guide/options/chain-hostname.md

@@ -0,0 +1,6 @@
+## chain-hostname()
+
+|   Type:|       `yes`, `no`|
+|Default:|              `no`|
+
+*Description:* This option can be used to enable or disable the chained hostname format. For more information, see the chain-hostnames() global option.

_includes/doc/admin-guide/options/config-kafka.md

@@ -0,0 +1,20 @@
+## config()
+
+|  Type: | key-value pairs |
+|Default:| N/A |
+
+*Description:* You can use this option to set the properties of the kafka {{ include.kafka_type }}.
+
+The {{ site.product.short_name }} kafka {{ include.type }} supports all properties of the official Kafka {{ include.kafka_type }}. For details, see the librdkafka documentation.
+
+The syntax of the config() option is the following:
+
+``` config
+config( 
+  “key1” => “value1” 
+  “key2” => “value2” 
+)
+```
+
+**NOTE:** The following kafka {{ include.kafka_type }} config options are protected and cannot be overriden in the `config()` list: {{ include.protected_options }}
+{: .notice--info}

_includes/doc/admin-guide/options/default-priority.md

@@ -1,8 +1,15 @@
+## default-level()
+
+This is just an alias of default-priority().
+
 ## default-priority()
 
 | Type:|  priority string |
 |  Default: | {{ page.priority_default | default: '' }} |
 
-*Description:* This parameter assigns an emergency level to the messages
-received from the {{ page.src | default: 'file' }} source if the message does not specify one. For
-example, default-priority(warning).
+*Description:* This option defines the default level value if the `PRIORITY` entry does not exist in the msg received from the {{ page.src | default: 'file' }} source.
+For example, `default-priority(warning)`.
+
+## default-severity()
+
+This is just an alias of default-priority().

_includes/doc/admin-guide/options/deprecated-options.md

@@ -1,2 +1,2 @@
-**NOTE:** The deprecated `{{ include.old }}` option is an alias for the `{{ include.new }}` option, retained for compatibility with earlier {{ site.product.short_name }} versions, but it may be removed at any time without further notice!
+**NOTE:** The deprecated `{{ include.old }}` option is an alias for the {{ include.new }} option, retained for compatibility with earlier {{ site.product.short_name }} versions, but it may be removed at any time without further notice!
 {: .notice--info}

_includes/doc/admin-guide/options/disable-bookmarks.md

@@ -0,0 +1,9 @@
+## disable-bookmarks()
+
+| Type:     | boolean |
+| Default:  | no |
+
+*Description:* This option prevents {{ site.product.short_name }} from storing a bookmark (such as position or offset) in its persist file for the last processed message.
+
+**NOTE:** This will not prevent usage of an already presented bookmark entry, for ignoring those bookmark entries specify `ignore-saved-bookmarks(yes)` as well.
+{: .notice--info}

_includes/doc/admin-guide/options/dns-cache.md

@@ -0,0 +1,6 @@
+## dns-cache()
+
+|   Type:|       `yes`, `no`|
+|Default:|              `no`|
+
+*Description:* This option enables or disables the DNS cache usage.

_includes/doc/admin-guide/options/ignore-saved-bookmarks.md

@@ -0,0 +1,6 @@
+## ignore-saved-bookmarks()
+
+|Type:|     boolean|
+|Default:| no|
+
+*Description:* By default, {{ site.product.short_name }} continues reading logs from the last remembered position (or offset, etc.) stored in its persist file after a restart or reload. If this option is set to `yes`, it will always start reading from either the beginning or the end of the available log list<!-- Intentionally no line and file ending, keep like that !!! -->

_includes/doc/admin-guide/options/kafka-logging.md

@@ -0,0 +1,13 @@
+## kafka-logging()
+
+| Accepted values: | disabled \| trace \| kafka |
+| Default:         | disabled |
+
+*Description:* This option allows you to control how internal Kafka logs appear in the {{ site.product.short_name }} logs.
+
+- disabled: Disables internal Kafka log messages in the {{ site.product.short_name }} logs.
+- trace: Logs all internal Kafka messages at the `trace` level of {{ site.product.short_name }}.
+- kafka: Logs internal Kafka messages using log levels mapped to those of {{ site.product.short_name }}.  
+
+**NOTE:** The internal Kafka logging level itself can be configured using the config() Kafka options. For details, refer to the librdkafka documentation.  
+{: .notice--info}

_includes/doc/admin-guide/options/kafka-source-workers.md

@@ -0,0 +1,6 @@
+- One **main worker** that fetches messages from the Kafka broker and stores them into an internal queue.
+- A second worker that processes the queued messages and forwards them to the configured destination.
+
+Although the source can operate using a single worker, this configuration typically results in a significant performance penalty compared to the default multi-worker setup.
+
+Increasing the number of workers beyond two may further improve throughput, especially when the main worker can fetch messages at high speed. In such cases, you may also need to fine-tune related options such as separated-worker-queues(), log-fetch-limit(), log-fetch-delay(), log-fetch-retry-delay(), log-fetch-queue-full-delay().

_includes/doc/admin-guide/options/log-fetch-delays.md

@@ -1,20 +1,20 @@
 ## log-fetch-delay()
 
-|Type:|     integer|
-|Default:| `10000`|
+|  Type: | integer [1 second / fetch_retry_delay * 1000000 milliseconds] |
+|Default:| 10000 (10 milliseconds)|
 
 *Description:* Sets the time {{ site.product.short_name }} waits between reading and sending log messages. The unit of this parameter is a fraction of a second, where `wait_time = 1 second / <defined value>`. For example, setting it to `1` results in approximately one log being read/sent per second, while `1000000` means a delay of only 1 microsecond between read/write attempts. The maximum value of this parameter is `1000000`.
 
 **NOTE:** Increasing the value of this parameter (which reduces the delay time) can improve log feed performance, but it may also increase system load.
 {: .notice--info}
 
-{% include doc/admin-guide/options/deprecated-options.md old='fetch-delay' new='log-fetch-delay' %}
+{% include doc/admin-guide/options/deprecated-options.md old='fetch-delay()' new='log-fetch-delay()' %}
 
 ## log-fetch-retry-delay()
 
-|Type:|     integer|
-|Default:| `1`|
+|Type:| integer in seconds |
+|Default:| 1|
 
 *Description:* Controls how many seconds {{ site.product.short_name }} remains idle before checking for new logs, in case no new logs were read during the previous check.
 
-{% include doc/admin-guide/options/deprecated-options.md old='fetch-retry-delay' new='log-fetch-retry-delay' %}
+{% include doc/admin-guide/options/deprecated-options.md old='fetch-retry-delay()' new='log-fetch-retry-delay()' %}

_includes/doc/admin-guide/options/log-prefix.md

@@ -1,11 +1,3 @@
-## log-prefix() (DEPRECATED)
+## log-prefix()
 
-|  Type: |     string|
-|Default:||
-
-*Description:* A string added to the beginning of every log message. It
-can be used to add an arbitrary string to any log source, though it is
-most commonly used for adding kernel: to the kernel messages on Linux.
-
-**NOTE:** This option is deprecated. Use **program-override** instead.
-{: .notice--info}
+{% include doc/admin-guide/options/deprecated-options.md old='log-prefix()' new='program-override()' %}

_includes/doc/admin-guide/options/read-old-records.md

@@ -0,0 +1,6 @@
+## read-old-records()
+
+|Type:|     boolean|
+|Default:| no|
+
+*Description:* If read-old-record() is set to `yes`, {{ site.product.short_name }} will start fetching from the oldest available message; otherwise, it will start from the newest one (if no bookmarks are present, or ignore-saved-bookmarks() is set to `yes`).

_sass/minimal-mistakes/minimal-mistakes/_variables.scss

@@ -45,11 +45,11 @@ $type-size-8: 0.625em !default; // ~10px
 $type-size-9: 0.5em !default; // ~8px
 
 /* headline scale */
-$h-size-1: 1.563em !default; // ~25.008px
-$h-size-2: 1.25em !default; // ~20px
-$h-size-3: 1.125em !default; // ~18px
-$h-size-4: 1.0625em !default; // ~17px
-$h-size-5: 1.03125em !default; // ~16.5px
+$h-size-1: 1.6em !default; // ~25.68px
+$h-size-2: 1.4em !default; // ~22.4px
+$h-size-3: 1.2em !default; // ~19.2px
+$h-size-4: 1.1em !default; // ~17.6px
+$h-size-5: 1.05em !default; // ~16.8px
 $h-size-6: 1em !default; // ~16px
 
 /*

doc/_admin-guide/060_Sources/020_File/README.md

@@ -48,6 +48,6 @@ source s_tail {
 
 **NOTE:** If the message does not have a proper syslog header, syslog-ng
 treats messages received from files as sent by the kern facility. Use
-the **default-facility()** and **default-priority()** options in the
+the default-facility() and default-priority() options in the
 source definition to assign a different facility if needed.
 {: .notice--info}

doc/_admin-guide/060_Sources/021_Wildcard-file/README.md

@@ -50,7 +50,7 @@ Note the following important points:
 
 - If the message does not have a proper syslog header, {{ site.product.short_name }}
     treats messages received from files as sent by the user facility.
-    Use the **default-facility()** and **default-priority()** options in
+    Use the default-facility() and default-priority() options in
     the source definition to assign a different facility if needed.
 
 - For every message that {{ site.product.short_name }} reads from the source files,