commit 779e50580c308e59ab781312615f431e31f5adc0 · pyrox.dev/nixpkgs

nixos/doc/manual/release-notes/rl-2311.section.md

···

       318
       318
        
       

     

       319
       319
        
       - The binary of the package `cloud-sql-proxy` has changed from `cloud_sql_proxy` to `cloud-sql-proxy`.

     

       320
       320
        
       

     

       321
       321
       +
       - The module `services.apache-kafka` was largely rewritten and has certain breaking changes. To be precise, this means that the following things have changed:

     

       322
       322
       +
       

     

       323
       323
       +
         - Most settings have been migrated to [services.apache-kafka.settings](#opt-services.apache-kafka.settings).

     

       324
       324
       +
           - Care must be taken when adapting an existing cluster to these changes, see [](#module-services-apache-kafka-migrating-to-settings).

     

       325
       325
       +
         - By virtue of being less opinionated, it is now possible to use the module to run Apache Kafka in KRaft mode instead of Zookeeper mode.

     

       326
       326
       +
           - [A few options](#module-services-apache-kafka-kraft) have been added to assist in this mode.

     

       327
       327
       +
       

     

       321
       328
        
       - Garage has been upgraded to 0.9.x. `services.garage.package` now needs to be explicitly set, so version upgrades can be done in a controlled fashion. For this, we expose `garage_x_y` attributes which can be set here.

     

       322
       329
        
       

     

       323
       330
        
       - `voms` and `xrootd` now moves the `$out/etc` content to the `$etc` output instead of `$out/etc.orig`, when input argument `externalEtc` is not `null`.

+130 -58

nixos/modules/services/misc/apache-kafka.nix

···

       5
       5
        
       let

     

       6
       6
        
         cfg = config.services.apache-kafka;

     

       7
       7
        
       

     

       8
       8
       -
         serverProperties =

     

       9
       9
       -
           if cfg.serverProperties != null then

     

       10
       10
       -
             cfg.serverProperties

     

       11
       11
       -
           else

     

       12
       12
       -
             ''

     

       13
       13
       -
               # Generated by nixos

     

       14
       14
       -
               broker.id=${toString cfg.brokerId}

     

       15
       15
       -
               port=${toString cfg.port}

     

       16
       16
       -
               host.name=${cfg.hostname}

     

       17
       17
       -
               log.dirs=${concatStringsSep "," cfg.logDirs}

     

       18
       18
       -
               zookeeper.connect=${cfg.zookeeper}

     

       19
       19
       -
               ${toString cfg.extraProperties}

     

       20
       20
       -
             '';

     

       8
       8
       +
         # The `javaProperties` generator takes care of various escaping rules and

     

       9
       9
       +
         # generation of the properties file, but we'll handle stringly conversion

     

       10
       10
       +
         # ourselves in mkPropertySettings and stringlySettings, since we know more

     

       11
       11
       +
         # about the specifically allowed format eg. for lists of this type, and we

     

       12
       12
       +
         # don't want to coerce-downsample values to str too early by having the

     

       13
       13
       +
         # coercedTypes from javaProperties directly in our NixOS option types.

     

       14
       14
       +
         #

     

       15
       15
       +
         # Make sure every `freeformType` and any specific option type in `settings` is

     

       16
       16
       +
         # supported here.

     

       17
       17
       +
       

     

       18
       18
       +
         mkPropertyString = let

     

       19
       19
       +
           render = {

     

       20
       20
       +
             bool = boolToString;

     

       21
       21
       +
             int = toString;

     

       22
       22
       +
             list = concatMapStringsSep "," mkPropertyString;

     

       23
       23
       +
             string = id;

     

       24
       24
       +
           };

     

       25
       25
       +
         in

     

       26
       26
       +
           v: render.${builtins.typeOf v} v;

     

       21
       27
        
       

     

       22
       22
       -
         serverConfig = pkgs.writeText "server.properties" serverProperties;

     

       23
       23
       -
         logConfig = pkgs.writeText "log4j.properties" cfg.log4jProperties;

     

       28
       28
       +
         stringlySettings = mapAttrs (_: mkPropertyString)

     

       29
       29
       +
           (filterAttrs (_: v:  v != null) cfg.settings);

     

       24
       30
        
       

     

       31
       31
       +
         generator = (pkgs.formats.javaProperties {}).generate;

     

       25
       32
        
       in {

     

       26
       33
        
       

     

       27
       34
        
         options.services.apache-kafka = {

     

       28
       28
       -
           enable = mkOption {

     

       29
       29
       -
             description = lib.mdDoc "Whether to enable Apache Kafka.";

     

       30
       30
       -
             default = false;

     

       31
       31
       -
             type = types.bool;

     

       32
       32
       -
           };

     

       35
       35
       +
           enable = mkEnableOption (lib.mdDoc "Apache Kafka event streaming broker");

     

       33
       36
        
       

     

       34
       34
       -
           brokerId = mkOption {

     

       35
       35
       -
             description = lib.mdDoc "Broker ID.";

     

       36
       36
       -
             default = -1;

     

       37
       37
       -
             type = types.int;

     

       38
       38
       -
           };

     

       37
       37
       +
           settings = mkOption {

     

       38
       38
       +
             description = lib.mdDoc ''

     

       39
       39
       +
               [Kafka broker configuration](https://kafka.apache.org/documentation.html#brokerconfigs)

     

       40
       40
       +
               {file}`server.properties`.

     

       41
       41
       +
       

     

       42
       42
       +
               Note that .properties files contain mappings from string to string.

     

       43
       43
       +
               Keys with dots are NOT represented by nested attrs in these settings,

     

       44
       44
       +
               but instead as quoted strings (ie. `settings."broker.id"`, NOT

     

       45
       45
       +
               `settings.broker.id`).

     

       46
       46
       +
            '';

     

       47
       47
       +
             type = types.submodule {

     

       48
       48
       +
               freeformType = with types; let

     

       49
       49
       +
                 primitive = oneOf [bool int str];

     

       50
       50
       +
               in lazyAttrsOf (nullOr (either primitive (listOf primitive)));

     

       51
       51
       +
       

     

       52
       52
       +
               options = {

     

       53
       53
       +
                 "broker.id" = mkOption {

     

       54
       54
       +
                   description = lib.mdDoc "Broker ID. -1 or null to auto-allocate in zookeeper mode.";

     

       55
       55
       +
                   default = null;

     

       56
       56
       +
                   type = with types; nullOr int;

     

       57
       57
       +
                 };

     

       58
       58
       +
       

     

       59
       59
       +
                 "log.dirs" = mkOption {

     

       60
       60
       +
                   description = lib.mdDoc "Log file directories.";

     

       61
       61
       +
                   # Deliberaly leave out old default and use the rewrite opportunity

     

       62
       62
       +
                   # to have users choose a safer value -- /tmp might be volatile and is a

     

       63
       63
       +
                   # slightly scary default choice.

     

       64
       64
       +
                   # default = [ "/tmp/apache-kafka" ];

     

       65
       65
       +
                   type = with types; listOf path;

     

       66
       66
       +
                 };

     

       39
       67
        
       

     

       40
       40
       -
           port = mkOption {

     

       41
       41
       -
             description = lib.mdDoc "Port number the broker should listen on.";

     

       42
       42
       -
             default = 9092;

     

       43
       43
       -
             type = types.port;

     

       68
       68
       +
                 "listeners" = mkOption {

     

       69
       69
       +
                   description = lib.mdDoc ''

     

       70
       70
       +
                     Kafka Listener List.

     

       71
       71
       +
                     See [listeners](https://kafka.apache.org/documentation/#brokerconfigs_listeners).

     

       72
       72
       +
                   '';

     

       73
       73
       +
                   type = types.listOf types.str;

     

       74
       74
       +
                   default = [ "PLAINTEXT://localhost:9092" ];

     

       75
       75
       +
                 };

     

       76
       76
       +
               };

     

       77
       77
       +
             };

     

       44
       78
        
           };

     

       45
       79
        
       

     

       46
       46
       -
           hostname = mkOption {

     

       47
       47
       -
             description = lib.mdDoc "Hostname the broker should bind to.";

     

       48
       48
       -
             default = "localhost";

     

       49
       49
       -
             type = types.str;

     

       80
       80
       +
           clusterId = mkOption {

     

       81
       81
       +
             description = lib.mdDoc ''

     

       82
       82
       +
               KRaft mode ClusterId used for formatting log directories. Can be generated with `kafka-storage.sh random-uuid`

     

       83
       83
       +
             '';

     

       84
       84
       +
             type = with types; nullOr str;

     

       85
       85
       +
             default = null;

     

       50
       86
        
           };

     

       51
       87
        
       

     

       52
       52
       -
           logDirs = mkOption {

     

       53
       53
       -
             description = lib.mdDoc "Log file directories";

     

       54
       54
       -
             default = [ "/tmp/kafka-logs" ];

     

       55
       55
       -
             type = types.listOf types.path;

     

       88
       88
       +
           configFiles.serverProperties = mkOption {

     

       89
       89
       +
             description = lib.mdDoc ''

     

       90
       90
       +
               Kafka server.properties configuration file path.

     

       91
       91
       +
               Defaults to the rendered `settings`.

     

       92
       92
       +
             '';

     

       93
       93
       +
             type = types.path;

     

       56
       94
        
           };

     

       57
       95
        
       

     

       58
       58
       -
           zookeeper = mkOption {

     

       59
       59
       -
             description = lib.mdDoc "Zookeeper connection string";

     

       60
       60
       -
             default = "localhost:2181";

     

       61
       61
       -
             type = types.str;

     

       96
       96
       +
           configFiles.log4jProperties = mkOption {

     

       97
       97
       +
             description = lib.mdDoc "Kafka log4j property configuration file path";

     

       98
       98
       +
             type = types.path;

     

       99
       99
       +
             default = pkgs.writeText "log4j.properties" cfg.log4jProperties;

     

       100
       100
       +
             defaultText = ''pkgs.writeText "log4j.properties" cfg.log4jProperties'';

     

       62
       101
        
           };

     

       63
       102
        
       

     

       64
       64
       -
           extraProperties = mkOption {

     

       65
       65
       -
             description = lib.mdDoc "Extra properties for server.properties.";

     

       66
       66
       -
             type = types.nullOr types.lines;

     

       67
       67
       -
             default = null;

     

       103
       103
       +
           formatLogDirs = mkOption {

     

       104
       104
       +
             description = lib.mdDoc ''

     

       105
       105
       +
               Whether to format log dirs in KRaft mode if all log dirs are

     

       106
       106
       +
               unformatted, ie. they contain no meta.properties.

     

       107
       107
       +
             '';

     

       108
       108
       +
             type = types.bool;

     

       109
       109
       +
             default = false;

     

       68
       110
        
           };

     

       69
       111
        
       

     

       70
       70
       -
           serverProperties = mkOption {

     

       112
       112
       +
           formatLogDirsIgnoreFormatted = mkOption {

     

       71
       113
        
             description = lib.mdDoc ''

     

       72
       72
       -
               Complete server.properties content. Other server.properties config

     

       73
       73
       -
               options will be ignored if this option is used.

     

       114
       114
       +
               Whether to ignore already formatted log dirs when formatting log dirs,

     

       115
       115
       +
               instead of failing. Useful when replacing or adding disks.

     

       74
       116
        
             '';

     

       75
       75
       -
             type = types.nullOr types.lines;

     

       76
       76
       -
             default = null;

     

       117
       117
       +
             type = types.bool;

     

       118
       118
       +
             default = false;

     

       77
       119
        
           };

     

       78
       120
        
       

     

       79
       121
        
           log4jProperties = mkOption {

     
···

       112
       154
        
             defaultText = literalExpression "pkgs.apacheKafka.passthru.jre";

     

       113
       155
        
             type = types.package;

     

       114
       156
        
           };

     

       157
       157
       +
         };

     

       158
       158
       +
       

     

       159
       159
       +
         imports = [

     

       160
       160
       +
           (mkRenamedOptionModule

     

       161
       161
       +
             [ "services" "apache-kafka" "brokerId" ]

     

       162
       162
       +
             [ "services" "apache-kafka" "settings" ''broker.id'' ])

     

       163
       163
       +
           (mkRenamedOptionModule

     

       164
       164
       +
             [ "services" "apache-kafka" "logDirs" ]

     

       165
       165
       +
             [ "services" "apache-kafka" "settings" ''log.dirs'' ])

     

       166
       166
       +
           (mkRenamedOptionModule

     

       167
       167
       +
             [ "services" "apache-kafka" "zookeeper" ]

     

       168
       168
       +
             [ "services" "apache-kafka" "settings" ''zookeeper.connect'' ])

     

       115
       169
        
       

     

       116
       116
       -
         };

     

       170
       170
       +
           (mkRemovedOptionModule [ "services" "apache-kafka" "port" ]

     

       171
       171
       +
             "Please see services.apache-kafka.settings.listeners and its documentation instead")

     

       172
       172
       +
           (mkRemovedOptionModule [ "services" "apache-kafka" "hostname" ]

     

       173
       173
       +
             "Please see services.apache-kafka.settings.listeners and its documentation instead")

     

       174
       174
       +
           (mkRemovedOptionModule [ "services" "apache-kafka" "extraProperties" ]

     

       175
       175
       +
             "Please see services.apache-kafka.settings and its documentation instead")

     

       176
       176
       +
           (mkRemovedOptionModule [ "services" "apache-kafka" "serverProperties" ]

     

       177
       177
       +
             "Please see services.apache-kafka.settings and its documentation instead")

     

       178
       178
       +
         ];

     

       117
       179
        
       

     

       118
       180
        
         config = mkIf cfg.enable {

     

       119
       119
       -
       

     

       120
       120
       -
           environment.systemPackages = [cfg.package];

     

       181
       181
       +
           services.apache-kafka.configFiles.serverProperties = generator "server.properties" stringlySettings;

     

       121
       182
        
       

     

       122
       183
        
           users.users.apache-kafka = {

     

       123
       184
        
             isSystemUser = true;

     

       124
       185
        
             group = "apache-kafka";

     

       125
       186
        
             description = "Apache Kafka daemon user";

     

       126
       126
       -
             home = head cfg.logDirs;

     

       127
       187
        
           };

     

       128
       188
        
           users.groups.apache-kafka = {};

     

       129
       189
        
       

     

       130
       130
       -
           systemd.tmpfiles.rules = map (logDir: "d '${logDir}' 0700 apache-kafka - - -") cfg.logDirs;

     

       190
       190
       +
           systemd.tmpfiles.rules = map (logDir: "d '${logDir}' 0700 apache-kafka - - -") cfg.settings."log.dirs";

     

       131
       191
        
       

     

       132
       192
        
           systemd.services.apache-kafka = {

     

       133
       193
        
             description = "Apache Kafka Daemon";

     

       134
       194
        
             wantedBy = [ "multi-user.target" ];

     

       135
       195
        
             after = [ "network.target" ];

     

       196
       196
       +
             preStart = mkIf cfg.formatLogDirs

     

       197
       197
       +
               (if cfg.formatLogDirsIgnoreFormatted then ''

     

       198
       198
       +
                 ${cfg.package}/bin/kafka-storage.sh format -t "${cfg.clusterId}" -c ${cfg.configFiles.serverProperties} --ignore-formatted

     

       199
       199
       +
               '' else ''

     

       200
       200
       +
                 if ${concatMapStringsSep " && " (l: ''[ ! -f "${l}/meta.properties" ]'') cfg.settings."log.dirs"}; then

     

       201
       201
       +
                   ${cfg.package}/bin/kafka-storage.sh format -t "${cfg.clusterId}" -c ${cfg.configFiles.serverProperties}

     

       202
       202
       +
                 fi

     

       203
       203
       +
               '');

     

       136
       204
        
             serviceConfig = {

     

       137
       205
        
               ExecStart = ''

     

       138
       206
        
                 ${cfg.jre}/bin/java \

     

       139
       207
        
                   -cp "${cfg.package}/libs/*" \

     

       140
       140
       -
                   -Dlog4j.configuration=file:${logConfig} \

     

       208
       208
       +
                   -Dlog4j.configuration=file:${cfg.configFiles.log4jProperties} \

     

       141
       209
        
                   ${toString cfg.jvmOptions} \

     

       142
       210
        
                   kafka.Kafka \

     

       143
       143
       -
                   ${serverConfig}

     

       211
       211
       +
                   ${cfg.configFiles.serverProperties}

     

       144
       212
        
               '';

     

       145
       213
        
               User = "apache-kafka";

     

       146
       214
        
               SuccessExitStatus = "0 143";

     

       147
       215
        
             };

     

       148
       216
        
           };

     

       217
       217
       +
         };

     

       149
       218
        
       

     

       150
       150
       -
         };

     

       219
       219
       +
         meta.doc = ./kafka.md;

     

       220
       220
       +
         meta.maintainers = with lib.maintainers; [

     

       221
       221
       +
           srhb

     

       222
       222
       +
         ];

     

       151
       223
        
       }

+63

nixos/modules/services/misc/kafka.md

···

       1
       1
       +
       # Apache Kafka {#module-services-apache-kafka}

     

       2
       2
       +
       

     

       3
       3
       +
       [Apache Kafka](https://kafka.apache.org/) is an open-source distributed event

     

       4
       4
       +
       streaming platform

     

       5
       5
       +
       

     

       6
       6
       +
       ## Basic Usage {#module-services-apache-kafka-basic-usage}

     

       7
       7
       +
       

     

       8
       8
       +
       The Apache Kafka service is configured almost exclusively through its

     

       9
       9
       +
       [settings](#opt-services.apache-kafka.settings) option, with each attribute

     

       10
       10
       +
       corresponding to the [upstream configuration

     

       11
       11
       +
       manual](https://kafka.apache.org/documentation/#configuration) broker settings.

     

       12
       12
       +
       

     

       13
       13
       +
       ## KRaft {#module-services-apache-kafka-kraft}

     

       14
       14
       +
       

     

       15
       15
       +
       Unlike in Zookeeper mode, Kafka in

     

       16
       16
       +
       [KRaft](https://kafka.apache.org/documentation/#kraft) mode requires each log

     

       17
       17
       +
       dir to be "formatted" (which means a cluster-specific a metadata file must

     

       18
       18
       +
       exist in each log dir)

     

       19
       19
       +
       

     

       20
       20
       +
       The upstream intention is for users to execute the [storage

     

       21
       21
       +
       tool](https://kafka.apache.org/documentation/#kraft_storage) to achieve this,

     

       22
       22
       +
       but this module contains a few extra options to automate this:

     

       23
       23
       +
       

     

       24
       24
       +
       - [](#opt-services.apache-kafka.clusterId)

     

       25
       25
       +
       - [](#opt-services.apache-kafka.formatLogDirs)

     

       26
       26
       +
       - [](#opt-services.apache-kafka.formatLogDirsIgnoreFormatted)

     

       27
       27
       +
       

     

       28
       28
       +
       ## Migrating to settings {#module-services-apache-kafka-migrating-to-settings}

     

       29
       29
       +
       

     

       30
       30
       +
       Migrating a cluster to the new `settings`-based changes requires adapting removed options to the corresponding upstream settings.

     

       31
       31
       +
       

     

       32
       32
       +
       This means that the upstream [Broker Configs documentation](https://kafka.apache.org/documentation/#brokerconfigs) should be followed closely.

     

       33
       33
       +
       

     

       34
       34
       +
       Note that dotted options in the upstream docs do _not_ correspond to nested Nix attrsets, but instead as quoted top level `settings` attributes, as in `services.apache-kafka.settings."broker.id"`, *NOT* `services.apache-kafka.settings.broker.id`.

     

       35
       35
       +
       

     

       36
       36
       +
       Care should be taken, especially when migrating clusters from the old module, to ensure that the same intended configuration is reproduced faithfully via `settings`.

     

       37
       37
       +
       

     

       38
       38
       +
       To assist in the comparison, the final config can be inspected by building the config file itself, ie. with: `nix-build <nixpkgs/nixos> -A config.services.apache-kafka.configFiles.serverProperties`.

     

       39
       39
       +
       

     

       40
       40
       +
       Notable changes to be aware of include:

     

       41
       41
       +
       

     

       42
       42
       +
       - Removal of `services.apache-kafka.extraProperties` and `services.apache-kafka.serverProperties`

     

       43
       43
       +
         - Translate using arbitrary properties using [](#opt-services.apache-kafka.settings)

     

       44
       44
       +
         - [Upstream docs](https://kafka.apache.org/documentation.html#brokerconfigs)

     

       45
       45
       +
         - The intention is for all broker properties to be fully representable via [](#opt-services.apache-kafka.settings).

     

       46
       46
       +
         - If this is not the case, please do consider raising an issue.

     

       47
       47
       +
         - Until it can be remedied, you *can* bail out by using [](#opt-services.apache-kafka.configFiles.serverProperties) to the path of a fully rendered properties file.

     

       48
       48
       +
       

     

       49
       49
       +
       - Removal of `services.apache-kafka.hostname` and `services.apache-kafka.port`

     

       50
       50
       +
         - Translate using: `services.apache-kafka.settings.listeners`

     

       51
       51
       +
         - [Upstream docs](https://kafka.apache.org/documentation.html#brokerconfigs_listeners)

     

       52
       52
       +
       

     

       53
       53
       +
       - Removal of `services.apache-kafka.logDirs`

     

       54
       54
       +
         - Translate using: `services.apache-kafka.settings."log.dirs"`

     

       55
       55
       +
         - [Upstream docs](https://kafka.apache.org/documentation.html#brokerconfigs_log.dirs)

     

       56
       56
       +
       

     

       57
       57
       +
       - Removal of `services.apache-kafka.brokerId`

     

       58
       58
       +
         - Translate using: `services.apache-kafka.settings."broker.id"`

     

       59
       59
       +
         - [Upstream docs](https://kafka.apache.org/documentation.html#brokerconfigs_broker.id)

     

       60
       60
       +
       

     

       61
       61
       +
       - Removal of `services.apache-kafka.zookeeper`

     

       62
       62
       +
         - Translate using: `services.apache-kafka.settings."zookeeper.connect"`

     

       63
       63
       +
         - [Upstream docs](https://kafka.apache.org/documentation.html#brokerconfigs_zookeeper.connect)

+61 -24

nixos/tests/kafka.nix

···

       6
       6
        
       with pkgs.lib;

     

       7
       7
        
       

     

       8
       8
        
       let

     

       9
       9
       -
         makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({

     

       9
       9
       +
         makeKafkaTest = name: { kafkaPackage, mode ? "zookeeper" }: (import ./make-test-python.nix ({

     

       10
       10
        
           inherit name;

     

       11
       11
        
           meta = with pkgs.lib.maintainers; {

     

       12
       12
        
             maintainers = [ nequissimus ];

     

       13
       13
        
           };

     

       14
       14
        
       

     

       15
       15
        
           nodes = {

     

       16
       16
       +
             kafka = { ... }: {

     

       17
       17
       +
               services.apache-kafka = mkMerge [

     

       18
       18
       +
                 ({

     

       19
       19
       +
                   enable = true;

     

       20
       20
       +
                   package = kafkaPackage;

     

       21
       21
       +
                   settings = {

     

       22
       22
       +
                     "offsets.topic.replication.factor" = 1;

     

       23
       23
       +
                     "log.dirs" = [

     

       24
       24
       +
                       "/var/lib/kafka/logdir1"

     

       25
       25
       +
                       "/var/lib/kafka/logdir2"

     

       26
       26
       +
                     ];

     

       27
       27
       +
                   };

     

       28
       28
       +
                 })

     

       29
       29
       +
                 (mkIf (mode == "zookeeper") {

     

       30
       30
       +
                   settings = {

     

       31
       31
       +
                     "zookeeper.session.timeout.ms" = 600000;

     

       32
       32
       +
                     "zookeeper.connect" = [ "zookeeper1:2181" ];

     

       33
       33
       +
                   };

     

       34
       34
       +
                 })

     

       35
       35
       +
                 (mkIf (mode == "kraft") {

     

       36
       36
       +
                   clusterId = "ak2fIHr4S8WWarOF_ODD0g";

     

       37
       37
       +
                   formatLogDirs = true;

     

       38
       38
       +
                   settings = {

     

       39
       39
       +
                     "node.id" = 1;

     

       40
       40
       +
                     "process.roles" = [

     

       41
       41
       +
                       "broker"

     

       42
       42
       +
                       "controller"

     

       43
       43
       +
                     ];

     

       44
       44
       +
                     "listeners" = [

     

       45
       45
       +
                       "PLAINTEXT://:9092"

     

       46
       46
       +
                       "CONTROLLER://:9093"

     

       47
       47
       +
                     ];

     

       48
       48
       +
                     "listener.security.protocol.map" = [

     

       49
       49
       +
                       "PLAINTEXT:PLAINTEXT"

     

       50
       50
       +
                       "CONTROLLER:PLAINTEXT"

     

       51
       51
       +
                     ];

     

       52
       52
       +
                     "controller.quorum.voters" = [

     

       53
       53
       +
                       "1@kafka:9093"

     

       54
       54
       +
                     ];

     

       55
       55
       +
                     "controller.listener.names" = [ "CONTROLLER" ];

     

       56
       56
       +
                   };

     

       57
       57
       +
                 })

     

       58
       58
       +
               ];

     

       59
       59
       +
       

     

       60
       60
       +
               networking.firewall.allowedTCPPorts = [ 9092 9093 ];

     

       61
       61
       +
               # i686 tests: qemu-system-i386 can simulate max 2047MB RAM (not 2048)

     

       62
       62
       +
               virtualisation.memorySize = 2047;

     

       63
       63
       +
             };

     

       64
       64
       +
           } // optionalAttrs (mode == "zookeeper") {

     

       16
       65
        
             zookeeper1 = { ... }: {

     

       17
       66
        
               services.zookeeper = {

     

       18
       67
        
                 enable = true;

     
···

       20
       69
        
       

     

       21
       70
        
               networking.firewall.allowedTCPPorts = [ 2181 ];

     

       22
       71
        
             };

     

       23
       23
       -
             kafka = { ... }: {

     

       24
       24
       -
               services.apache-kafka = {

     

       25
       25
       -
                 enable = true;

     

       26
       26
       -
                 extraProperties = ''

     

       27
       27
       -
                   offsets.topic.replication.factor = 1

     

       28
       28
       -
                   zookeeper.session.timeout.ms = 600000

     

       29
       29
       -
                 '';

     

       30
       30
       -
                 package = kafkaPackage;

     

       31
       31
       -
                 zookeeper = "zookeeper1:2181";

     

       32
       32
       -
               };

     

       33
       33
       -
       

     

       34
       34
       -
               networking.firewall.allowedTCPPorts = [ 9092 ];

     

       35
       35
       -
               # i686 tests: qemu-system-i386 can simulate max 2047MB RAM (not 2048)

     

       36
       36
       -
               virtualisation.memorySize = 2047;

     

       37
       37
       -
             };

     

       38
       72
        
           };

     

       39
       73
        
       

     

       40
       74
        
           testScript = ''

     

       41
       75
        
             start_all()

     

       42
       76
        
       

     

       77
       77
       +
             ${optionalString (mode == "zookeeper") ''

     

       43
       78
        
             zookeeper1.wait_for_unit("default.target")

     

       44
       79
        
             zookeeper1.wait_for_unit("zookeeper.service")

     

       45
       80
        
             zookeeper1.wait_for_open_port(2181)

     

       81
       81
       +
             ''}

     

       46
       82
        
       

     

       47
       83
        
             kafka.wait_for_unit("default.target")

     

       48
       84
        
             kafka.wait_for_unit("apache-kafka.service")

     
···

       67
       103
        
         }) { inherit system; });

     

       68
       104
        
       

     

       69
       105
        
       in with pkgs; {

     

       70
       70
       -
         kafka_2_8  = makeKafkaTest "kafka_2_8"  apacheKafka_2_8;

     

       71
       71
       -
         kafka_3_0  = makeKafkaTest "kafka_3_0"  apacheKafka_3_0;

     

       72
       72
       -
         kafka_3_1  = makeKafkaTest "kafka_3_1"  apacheKafka_3_1;

     

       73
       73
       -
         kafka_3_2  = makeKafkaTest "kafka_3_2"  apacheKafka_3_2;

     

       74
       74
       -
         kafka_3_3  = makeKafkaTest "kafka_3_3"  apacheKafka_3_3;

     

       75
       75
       -
         kafka_3_4  = makeKafkaTest "kafka_3_4"  apacheKafka_3_4;

     

       76
       76
       -
         kafka_3_5  = makeKafkaTest "kafka_3_5"  apacheKafka_3_5;

     

       77
       77
       -
         kafka  = makeKafkaTest "kafka"  apacheKafka;

     

       106
       106
       +
         kafka_2_8 = makeKafkaTest "kafka_2_8" { kafkaPackage = apacheKafka_2_8; };

     

       107
       107
       +
         kafka_3_0 = makeKafkaTest "kafka_3_0" { kafkaPackage = apacheKafka_3_0; };

     

       108
       108
       +
         kafka_3_1 = makeKafkaTest "kafka_3_1" { kafkaPackage = apacheKafka_3_1; };

     

       109
       109
       +
         kafka_3_2 = makeKafkaTest "kafka_3_2" { kafkaPackage = apacheKafka_3_2; };

     

       110
       110
       +
         kafka_3_3 = makeKafkaTest "kafka_3_3" { kafkaPackage = apacheKafka_3_3; };

     

       111
       111
       +
         kafka_3_4 = makeKafkaTest "kafka_3_4" { kafkaPackage = apacheKafka_3_4; };

     

       112
       112
       +
         kafka_3_5 = makeKafkaTest "kafka_3_5" { kafkaPackage = apacheKafka_3_5; };

     

       113
       113
       +
         kafka = makeKafkaTest "kafka" { kafkaPackage = apacheKafka; };

     

       114
       114
       +
         kafka_kraft = makeKafkaTest "kafka_kraft" { kafkaPackage = apacheKafka; mode = "kraft"; };

     

       78
       115
        
       }