init

venv/lib/python3.11/site-packages/kafka/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+
+__title__ = 'kafka'
+from kafka.version import __version__
+__author__ = 'Dana Powers'
+__license__ = 'Apache License 2.0'
+__copyright__ = 'Copyright 2025 Dana Powers, David Arthur, and Contributors'
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+try:  # Python 2.7+
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass
+
+logging.getLogger(__name__).addHandler(NullHandler())
+
+
+from kafka.admin import KafkaAdminClient
+from kafka.client_async import KafkaClient
+from kafka.consumer import KafkaConsumer
+from kafka.consumer.subscription_state import ConsumerRebalanceListener
+from kafka.producer import KafkaProducer
+from kafka.conn import BrokerConnection
+from kafka.serializer import Serializer, Deserializer
+from kafka.structs import TopicPartition, OffsetAndMetadata
+
+
+__all__ = [
+    'BrokerConnection', 'ConsumerRebalanceListener', 'KafkaAdminClient',
+    'KafkaClient', 'KafkaConsumer', 'KafkaProducer',
+]

venv/lib/python3.11/site-packages/kafka/admin/__init__.py

@@ -0,0 +1,14 @@
+from __future__ import absolute_import
+
+from kafka.admin.config_resource import ConfigResource, ConfigResourceType
+from kafka.admin.client import KafkaAdminClient
+from kafka.admin.acl_resource import (ACL, ACLFilter, ResourcePattern, ResourcePatternFilter, ACLOperation,
+                                      ResourceType, ACLPermissionType, ACLResourcePatternType)
+from kafka.admin.new_topic import NewTopic
+from kafka.admin.new_partitions import NewPartitions
+
+__all__ = [
+    'ConfigResource', 'ConfigResourceType', 'KafkaAdminClient', 'NewTopic', 'NewPartitions', 'ACL', 'ACLFilter',
+    'ResourcePattern', 'ResourcePatternFilter', 'ACLOperation', 'ResourceType', 'ACLPermissionType',
+    'ACLResourcePatternType'
+]

venv/lib/python3.11/site-packages/kafka/admin/acl_resource.py

@@ -0,0 +1,244 @@
+from __future__ import absolute_import
+from kafka.errors import IllegalArgumentError
+
+# enum in stdlib as of py3.4
+try:
+    from enum import IntEnum  # pylint: disable=import-error
+except ImportError:
+    # vendored backport module
+    from kafka.vendor.enum34 import IntEnum
+
+
+class ResourceType(IntEnum):
+    """Type of kafka resource to set ACL for
+
+    The ANY value is only valid in a filter context
+    """
+
+    UNKNOWN = 0,
+    ANY = 1,
+    CLUSTER = 4,
+    DELEGATION_TOKEN = 6,
+    GROUP = 3,
+    TOPIC = 2,
+    TRANSACTIONAL_ID = 5
+
+
+class ACLOperation(IntEnum):
+    """Type of operation
+
+    The ANY value is only valid in a filter context
+    """
+
+    ANY = 1,
+    ALL = 2,
+    READ = 3,
+    WRITE = 4,
+    CREATE = 5,
+    DELETE = 6,
+    ALTER = 7,
+    DESCRIBE = 8,
+    CLUSTER_ACTION = 9,
+    DESCRIBE_CONFIGS = 10,
+    ALTER_CONFIGS = 11,
+    IDEMPOTENT_WRITE = 12
+
+
+class ACLPermissionType(IntEnum):
+    """An enumerated type of permissions
+
+    The ANY value is only valid in a filter context
+    """
+
+    ANY = 1,
+    DENY = 2,
+    ALLOW = 3
+
+
+class ACLResourcePatternType(IntEnum):
+    """An enumerated type of resource patterns
+
+    More details on the pattern types and how they work
+    can be found in KIP-290 (Support for prefixed ACLs)
+    https://cwiki.apache.org/confluence/display/KAFKA/KIP-290%3A+Support+for+Prefixed+ACLs
+    """
+
+    ANY = 1,
+    MATCH = 2,
+    LITERAL = 3,
+    PREFIXED = 4
+
+
+class ACLFilter(object):
+    """Represents a filter to use with describing and deleting ACLs
+
+    The difference between this class and the ACL class is mainly that
+    we allow using ANY with the operation, permission, and resource type objects
+    to fetch ALCs matching any of the properties.
+
+    To make a filter matching any principal, set principal to None
+    """
+
+    def __init__(
+        self,
+        principal,
+        host,
+        operation,
+        permission_type,
+        resource_pattern
+    ):
+        self.principal = principal
+        self.host = host
+        self.operation = operation
+        self.permission_type = permission_type
+        self.resource_pattern = resource_pattern
+
+        self.validate()
+
+    def validate(self):
+        if not isinstance(self.operation, ACLOperation):
+            raise IllegalArgumentError("operation must be an ACLOperation object, and cannot be ANY")
+        if not isinstance(self.permission_type, ACLPermissionType):
+            raise IllegalArgumentError("permission_type must be an ACLPermissionType object, and cannot be ANY")
+        if not isinstance(self.resource_pattern, ResourcePatternFilter):
+            raise IllegalArgumentError("resource_pattern must be a ResourcePatternFilter object")
+
+    def __repr__(self):
+        return "<ACL principal={principal}, resource={resource}, operation={operation}, type={type}, host={host}>".format(
+            principal=self.principal,
+            host=self.host,
+            operation=self.operation.name,
+            type=self.permission_type.name,
+            resource=self.resource_pattern
+        )
+
+    def __eq__(self, other):
+        return all((
+            self.principal == other.principal,
+            self.host == other.host,
+            self.operation == other.operation,
+            self.permission_type == other.permission_type,
+            self.resource_pattern == other.resource_pattern
+        ))
+
+    def __hash__(self):
+        return hash((
+            self.principal,
+            self.host,
+            self.operation,
+            self.permission_type,
+            self.resource_pattern,
+        ))
+
+
+class ACL(ACLFilter):
+    """Represents a concrete ACL for a specific ResourcePattern
+
+    In kafka an ACL is a 4-tuple of (principal, host, operation, permission_type)
+    that limits who can do what on a specific resource (or since KIP-290 a resource pattern)
+
+    Terminology:
+    Principal -> This is the identifier for the user. Depending on the authorization method used (SSL, SASL etc)
+        the principal will look different. See http://kafka.apache.org/documentation/#security_authz for details.
+        The principal must be on the format "User:<name>" or kafka will treat it as invalid. It's possible to use
+        other principal types than "User" if using a custom authorizer for the cluster.
+    Host -> This must currently be an IP address. It cannot be a range, and it cannot be a domain name.
+        It can be set to "*", which is special cased in kafka to mean "any host"
+    Operation -> Which client operation this ACL refers to. Has different meaning depending
+        on the resource type the ACL refers to. See https://docs.confluent.io/current/kafka/authorization.html#acl-format
+        for a list of which combinations of resource/operation that unlocks which kafka APIs
+    Permission Type: Whether this ACL is allowing or denying access
+    Resource Pattern -> This is a representation of the resource or resource pattern that the ACL
+        refers to. See the ResourcePattern class for details.
+
+    """
+
+    def __init__(
+            self,
+            principal,
+            host,
+            operation,
+            permission_type,
+            resource_pattern
+    ):
+        super(ACL, self).__init__(principal, host, operation, permission_type, resource_pattern)
+        self.validate()
+
+    def validate(self):
+        if self.operation == ACLOperation.ANY:
+            raise IllegalArgumentError("operation cannot be ANY")
+        if self.permission_type == ACLPermissionType.ANY:
+            raise IllegalArgumentError("permission_type cannot be ANY")
+        if not isinstance(self.resource_pattern, ResourcePattern):
+            raise IllegalArgumentError("resource_pattern must be a ResourcePattern object")
+
+
+class ResourcePatternFilter(object):
+    def __init__(
+            self,
+            resource_type,
+            resource_name,
+            pattern_type
+    ):
+        self.resource_type = resource_type
+        self.resource_name = resource_name
+        self.pattern_type = pattern_type
+
+        self.validate()
+
+    def validate(self):
+        if not isinstance(self.resource_type, ResourceType):
+            raise IllegalArgumentError("resource_type must be a ResourceType object")
+        if not isinstance(self.pattern_type, ACLResourcePatternType):
+            raise IllegalArgumentError("pattern_type must be an ACLResourcePatternType object")
+
+    def __repr__(self):
+        return "<ResourcePattern type={}, name={}, pattern={}>".format(
+            self.resource_type.name,
+            self.resource_name,
+            self.pattern_type.name
+        )
+
+    def __eq__(self, other):
+        return all((
+            self.resource_type == other.resource_type,
+            self.resource_name == other.resource_name,
+            self.pattern_type == other.pattern_type,
+        ))
+
+    def __hash__(self):
+        return hash((
+            self.resource_type,
+            self.resource_name,
+            self.pattern_type
+        ))
+
+
+class ResourcePattern(ResourcePatternFilter):
+    """A resource pattern to apply the ACL to
+
+    Resource patterns are used to be able to specify which resources an ACL
+    describes in a more flexible way than just pointing to a literal topic name for example.
+    Since KIP-290 (kafka 2.0) it's possible to set an ACL for a prefixed resource name, which
+    can cut down considerably on the number of ACLs needed when the number of topics and
+    consumer groups start to grow.
+    The default pattern_type is LITERAL, and it describes a specific resource. This is also how
+    ACLs worked before the introduction of prefixed ACLs
+    """
+
+    def __init__(
+            self,
+            resource_type,
+            resource_name,
+            pattern_type=ACLResourcePatternType.LITERAL
+    ):
+        super(ResourcePattern, self).__init__(resource_type, resource_name, pattern_type)
+        self.validate()
+
+    def validate(self):
+        if self.resource_type == ResourceType.ANY:
+            raise IllegalArgumentError("resource_type cannot be ANY")
+        if self.pattern_type in [ACLResourcePatternType.ANY, ACLResourcePatternType.MATCH]:
+            raise IllegalArgumentError(
+                "pattern_type cannot be {} on a concrete ResourcePattern".format(self.pattern_type.name)
+            )

venv/lib/python3.11/site-packages/kafka/admin/config_resource.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import
+
+# enum in stdlib as of py3.4
+try:
+    from enum import IntEnum  # pylint: disable=import-error
+except ImportError:
+    # vendored backport module
+    from kafka.vendor.enum34 import IntEnum
+
+
+class ConfigResourceType(IntEnum):
+    """An enumerated type of config resources"""
+
+    BROKER = 4,
+    TOPIC = 2
+
+
+class ConfigResource(object):
+    """A class for specifying config resources.
+    Arguments:
+        resource_type (ConfigResourceType): the type of kafka resource
+        name (string): The name of the kafka resource
+        configs ({key : value}): A  maps of config keys to values.
+    """
+
+    def __init__(
+            self,
+            resource_type,
+            name,
+            configs=None
+    ):
+        if not isinstance(resource_type, (ConfigResourceType)):
+            resource_type = ConfigResourceType[str(resource_type).upper()] # pylint: disable-msg=unsubscriptable-object
+        self.resource_type = resource_type
+        self.name = name
+        self.configs = configs

venv/lib/python3.11/site-packages/kafka/admin/new_partitions.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+
+class NewPartitions(object):
+    """A class for new partition creation on existing topics.  Note that the length of new_assignments, if specified,
+    must be the difference between the new total number of partitions and the existing number of partitions.
+    Arguments:
+        total_count (int): the total number of partitions that should exist on the topic
+        new_assignments ([[int]]): an array of arrays of replica assignments for new partitions.
+            If not set, broker assigns replicas per an internal algorithm.
+    """
+
+    def __init__(
+        self,
+        total_count,
+        new_assignments=None
+    ):
+        self.total_count = total_count
+        self.new_assignments = new_assignments

venv/lib/python3.11/site-packages/kafka/admin/new_topic.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+
+from kafka.errors import IllegalArgumentError
+
+
+class NewTopic(object):
+    """ A class for new topic creation
+    Arguments:
+        name (string): name of the topic
+        num_partitions (int): number of partitions
+            or -1 if replica_assignment has been specified
+        replication_factor (int): replication factor or -1 if
+            replica assignment is specified
+        replica_assignment (dict of int: [int]): A mapping containing
+            partition id and replicas to assign to it.
+        topic_configs (dict of str: str): A mapping of config key
+            and value for the topic.
+    """
+
+    def __init__(
+            self,
+            name,
+            num_partitions,
+            replication_factor,
+            replica_assignments=None,
+            topic_configs=None,
+    ):
+        if not (num_partitions == -1 or replication_factor == -1) ^ (replica_assignments is None):
+            raise IllegalArgumentError('either num_partitions/replication_factor or replica_assignment must be specified')
+        self.name = name
+        self.num_partitions = num_partitions
+        self.replication_factor = replication_factor
+        self.replica_assignments = replica_assignments or {}
+        self.topic_configs = topic_configs or {}

venv/lib/python3.11/site-packages/kafka/benchmarks/consumer_performance.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# Adapted from https://github.com/mrafayaleem/kafka-jython
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import pprint
+import sys
+import threading
+import time
+import traceback
+
+from kafka import KafkaConsumer
+
+
+class ConsumerPerformance(object):
+    @staticmethod
+    def run(args):
+        try:
+            props = {}
+            for prop in args.consumer_config:
+                k, v = prop.split('=')
+                try:
+                    v = int(v)
+                except ValueError:
+                    pass
+                if v == 'None':
+                    v = None
+                elif v == 'False':
+                    v = False
+                elif v == 'True':
+                    v = True
+                props[k] = v
+
+            print('Initializing Consumer...')
+            props['bootstrap_servers'] = args.bootstrap_servers
+            props['auto_offset_reset'] = 'earliest'
+            if 'group_id' not in props:
+                props['group_id'] = 'kafka-consumer-benchmark'
+            if 'consumer_timeout_ms' not in props:
+                props['consumer_timeout_ms'] = 10000
+            props['metrics_sample_window_ms'] = args.stats_interval * 1000
+            for k, v in props.items():
+                print('---> {0}={1}'.format(k, v))
+            consumer = KafkaConsumer(args.topic, **props)
+            print('---> group_id={0}'.format(consumer.config['group_id']))
+            print('---> report stats every {0} secs'.format(args.stats_interval))
+            print('---> raw metrics? {0}'.format(args.raw_metrics))
+            timer_stop = threading.Event()
+            timer = StatsReporter(args.stats_interval, consumer,
+                                  event=timer_stop,
+                                  raw_metrics=args.raw_metrics)
+            timer.start()
+            print('-> OK!')
+            print()
+
+            start_time = time.time()
+            records = 0
+            for msg in consumer:
+                records += 1
+                if records >= args.num_records:
+                    break
+
+            end_time = time.time()
+            timer_stop.set()
+            timer.join()
+            print('Consumed {0} records'.format(records))
+            print('Execution time:', end_time - start_time, 'secs')
+
+        except Exception:
+            exc_info = sys.exc_info()
+            traceback.print_exception(*exc_info)
+            sys.exit(1)
+
+
+class StatsReporter(threading.Thread):
+    def __init__(self, interval, consumer, event=None, raw_metrics=False):
+        super(StatsReporter, self).__init__()
+        self.interval = interval
+        self.consumer = consumer
+        self.event = event
+        self.raw_metrics = raw_metrics
+
+    def print_stats(self):
+        metrics = self.consumer.metrics()
+        if self.raw_metrics:
+            pprint.pprint(metrics)
+        else:
+            print('{records-consumed-rate} records/sec ({bytes-consumed-rate} B/sec),'
+                  ' {fetch-latency-avg} latency,'
+                  ' {fetch-rate} fetch/s,'
+                  ' {fetch-size-avg} fetch size,'
+                  ' {records-lag-max} max record lag,'
+                  ' {records-per-request-avg} records/req'
+                  .format(**metrics['consumer-fetch-manager-metrics']))
+
+
+    def print_final(self):
+        self.print_stats()
+
+    def run(self):
+        while self.event and not self.event.wait(self.interval):
+            self.print_stats()
+        else:
+            self.print_final()
+
+
+def get_args_parser():
+    parser = argparse.ArgumentParser(
+        description='This tool is used to verify the consumer performance.')
+
+    parser.add_argument(
+        '--bootstrap-servers', type=str, nargs='+', default=(),
+        help='host:port for cluster bootstrap servers')
+    parser.add_argument(
+        '--topic', type=str,
+        help='Topic for consumer test (default: kafka-python-benchmark-test)',
+        default='kafka-python-benchmark-test')
+    parser.add_argument(
+        '--num-records', type=int,
+        help='number of messages to consume (default: 1000000)',
+        default=1000000)
+    parser.add_argument(
+        '--consumer-config', type=str, nargs='+', default=(),
+        help='kafka consumer related configuration properties like '
+             'bootstrap_servers,client_id etc..')
+    parser.add_argument(
+        '--fixture-compression', type=str,
+        help='specify a compression type for use with broker fixtures / producer')
+    parser.add_argument(
+        '--stats-interval', type=int,
+        help='Interval in seconds for stats reporting to console (default: 5)',
+        default=5)
+    parser.add_argument(
+        '--raw-metrics', action='store_true',
+        help='Enable this flag to print full metrics dict on each interval')
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_args_parser().parse_args()
+    ConsumerPerformance.run(args)

venv/lib/python3.11/site-packages/kafka/benchmarks/load_example.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import logging
+import threading
+import time
+
+from kafka import KafkaConsumer, KafkaProducer
+
+
+class Producer(threading.Thread):
+
+    def __init__(self, bootstrap_servers, topic, stop_event, msg_size):
+        super(Producer, self).__init__()
+        self.bootstrap_servers = bootstrap_servers
+        self.topic = topic
+        self.stop_event = stop_event
+        self.big_msg = b'1' * msg_size
+
+    def run(self):
+        producer = KafkaProducer(bootstrap_servers=self.bootstrap_servers)
+        self.sent = 0
+
+        while not self.stop_event.is_set():
+            producer.send(self.topic, self.big_msg)
+            self.sent += 1
+        producer.flush()
+        producer.close()
+
+
+class Consumer(threading.Thread):
+    def __init__(self, bootstrap_servers, topic, stop_event, msg_size):
+        super(Consumer, self).__init__()
+        self.bootstrap_servers = bootstrap_servers
+        self.topic = topic
+        self.stop_event = stop_event
+        self.msg_size = msg_size
+
+    def run(self):
+        consumer = KafkaConsumer(bootstrap_servers=self.bootstrap_servers,
+                                 auto_offset_reset='earliest')
+        consumer.subscribe([self.topic])
+        self.valid = 0
+        self.invalid = 0
+
+        for message in consumer:
+            if len(message.value) == self.msg_size:
+                self.valid += 1
+            else:
+                print('Invalid message:', len(message.value), self.msg_size)
+                self.invalid += 1
+
+            if self.stop_event.is_set():
+                break
+        consumer.close()
+
+
+def get_args_parser():
+    parser = argparse.ArgumentParser(
+        description='This tool is used to demonstrate consumer and producer load.')
+
+    parser.add_argument(
+        '--bootstrap-servers', type=str, nargs='+', default=('localhost:9092'),
+        help='host:port for cluster bootstrap servers (default: localhost:9092)')
+    parser.add_argument(
+        '--topic', type=str,
+        help='Topic for load test (default: kafka-python-benchmark-load-example)',
+        default='kafka-python-benchmark-load-example')
+    parser.add_argument(
+        '--msg-size', type=int,
+        help='Message size, in bytes, for load test (default: 524288)',
+        default=524288)
+    parser.add_argument(
+        '--load-time', type=int,
+        help='number of seconds to run load test (default: 10)',
+        default=10)
+    parser.add_argument(
+        '--log-level', type=str,
+        help='Optional logging level for load test: ERROR|INFO|DEBUG etc',
+        default=None)
+    return parser
+
+
+def main(args):
+    if args.log_level:
+        logging.basicConfig(
+            format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
+            level=getattr(logging, args.log_level))
+    producer_stop = threading.Event()
+    consumer_stop = threading.Event()
+    threads = [
+        Producer(args.bootstrap_servers, args.topic, producer_stop, args.msg_size),
+        Consumer(args.bootstrap_servers, args.topic, consumer_stop, args.msg_size)
+    ]
+
+    for t in threads:
+        t.start()
+
+    time.sleep(args.load_time)
+    producer_stop.set()
+    consumer_stop.set()
+    print('Messages sent: %d' % threads[0].sent)
+    print('Messages recvd: %d' % threads[1].valid)
+    print('Messages invalid: %d' % threads[1].invalid)
+
+
+if __name__ == "__main__":
+    args = get_args_parser().parse_args()
+    main(args)

venv/lib/python3.11/site-packages/kafka/benchmarks/producer_performance.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# Adapted from https://github.com/mrafayaleem/kafka-jython
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import pprint
+import sys
+import threading
+import time
+import traceback
+
+from kafka.vendor.six.moves import range
+
+from kafka import KafkaProducer
+
+
+class ProducerPerformance(object):
+    @staticmethod
+    def run(args):
+        try:
+            props = {}
+            for prop in args.producer_config:
+                k, v = prop.split('=')
+                try:
+                    v = int(v)
+                except ValueError:
+                    pass
+                if v == 'None':
+                    v = None
+                elif v == 'False':
+                    v = False
+                elif v == 'True':
+                    v = True
+                props[k] = v
+
+            print('Initializing producer...')
+            props['bootstrap_servers'] = args.bootstrap_servers
+            record = bytes(bytearray(args.record_size))
+            props['metrics_sample_window_ms'] = args.stats_interval * 1000
+
+            producer = KafkaProducer(**props)
+            for k, v in props.items():
+                print('---> {0}={1}'.format(k, v))
+            print('---> send {0} byte records'.format(args.record_size))
+            print('---> report stats every {0} secs'.format(args.stats_interval))
+            print('---> raw metrics? {0}'.format(args.raw_metrics))
+            timer_stop = threading.Event()
+            timer = StatsReporter(args.stats_interval, producer,
+                                  event=timer_stop,
+                                  raw_metrics=args.raw_metrics)
+            timer.start()
+            print('-> OK!')
+            print()
+
+            def _benchmark():
+                results = []
+                for i in range(args.num_records):
+                    results.append(producer.send(topic=args.topic, value=record))
+                print("Send complete...")
+                producer.flush()
+                producer.close()
+                count_success, count_failure = 0, 0
+                for r in results:
+                    if r.succeeded():
+                        count_success += 1
+                    elif r.failed():
+                        count_failure += 1
+                    else:
+                        raise ValueError(r)
+                print("%d suceeded, %d failed" % (count_success, count_failure))
+
+            start_time = time.time()
+            _benchmark()
+            end_time = time.time()
+            timer_stop.set()
+            timer.join()
+            print('Execution time:', end_time - start_time, 'secs')
+
+        except Exception:
+            exc_info = sys.exc_info()
+            traceback.print_exception(*exc_info)
+            sys.exit(1)
+
+
+class StatsReporter(threading.Thread):
+    def __init__(self, interval, producer, event=None, raw_metrics=False):
+        super(StatsReporter, self).__init__()
+        self.interval = interval
+        self.producer = producer
+        self.event = event
+        self.raw_metrics = raw_metrics
+
+    def print_stats(self):
+        metrics = self.producer.metrics()
+        if not metrics:
+            return
+        if self.raw_metrics:
+            pprint.pprint(metrics)
+        else:
+            print('{record-send-rate} records/sec ({byte-rate} B/sec),'
+                  ' {request-latency-avg} latency,'
+                  ' {record-size-avg} record size,'
+                  ' {batch-size-avg} batch size,'
+                  ' {records-per-request-avg} records/req'
+                  .format(**metrics['producer-metrics']))
+
+    def print_final(self):
+        self.print_stats()
+
+    def run(self):
+        while self.event and not self.event.wait(self.interval):
+            self.print_stats()
+        else:
+            self.print_final()
+
+
+def get_args_parser():
+    parser = argparse.ArgumentParser(
+        description='This tool is used to verify the producer performance.')
+
+    parser.add_argument(
+        '--bootstrap-servers', type=str, nargs='+', default=(),
+        help='host:port for cluster bootstrap server')
+    parser.add_argument(
+        '--topic', type=str,
+        help='Topic name for test (default: kafka-python-benchmark-test)',
+        default='kafka-python-benchmark-test')
+    parser.add_argument(
+        '--num-records', type=int,
+        help='number of messages to produce (default: 1000000)',
+        default=1000000)
+    parser.add_argument(
+        '--record-size', type=int,
+        help='message size in bytes (default: 100)',
+        default=100)
+    parser.add_argument(
+        '--producer-config', type=str, nargs='+', default=(),
+        help='kafka producer related configuaration properties like '
+             'bootstrap_servers,client_id etc..')
+    parser.add_argument(
+        '--stats-interval', type=int,
+        help='Interval in seconds for stats reporting to console (default: 5)',
+        default=5)
+    parser.add_argument(
+        '--raw-metrics', action='store_true',
+        help='Enable this flag to print full metrics dict on each interval')
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_args_parser().parse_args()
+    ProducerPerformance.run(args)

venv/lib/python3.11/site-packages/kafka/benchmarks/record_batch_compose.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+from __future__ import print_function
+import hashlib
+import itertools
+import os
+import random
+
+import pyperf
+
+from kafka.record.memory_records import MemoryRecordsBuilder
+
+
+DEFAULT_BATCH_SIZE = 1600 * 1024
+KEY_SIZE = 6
+VALUE_SIZE = 60
+TIMESTAMP_RANGE = [1505824130000, 1505824140000]
+
+# With values above v1 record is 100 bytes, so 10 000 bytes for 100 messages
+MESSAGES_PER_BATCH = 100
+
+
+def random_bytes(length):
+    buffer = bytearray(length)
+    for i in range(length):
+        buffer[i] = random.randint(0, 255)
+    return bytes(buffer)
+
+
+def prepare():
+    return iter(itertools.cycle([
+        (random_bytes(KEY_SIZE),
+         random_bytes(VALUE_SIZE),
+         random.randint(*TIMESTAMP_RANGE)
+         )
+        for _ in range(int(MESSAGES_PER_BATCH * 1.94))
+    ]))
+
+
+def finalize(results):
+    # Just some strange code to make sure PyPy does execute the main code
+    # properly, without optimizing it away
+    hash_val = hashlib.md5()
+    for buf in results:
+        hash_val.update(buf)
+    print(hash_val, file=open(os.devnull, "w"))
+
+
+def func(loops, magic):
+    # Jit can optimize out the whole function if the result is the same each
+    # time, so we need some randomized input data )
+    precomputed_samples = prepare()
+    results = []
+
+    # Main benchmark code.
+    t0 = pyperf.perf_counter()
+    for _ in range(loops):
+        batch = MemoryRecordsBuilder(
+            magic, batch_size=DEFAULT_BATCH_SIZE, compression_type=0)
+        for _ in range(MESSAGES_PER_BATCH):
+            key, value, timestamp = next(precomputed_samples)
+            size = batch.append(
+                timestamp=timestamp, key=key, value=value)
+            assert size
+        batch.close()
+        results.append(batch.buffer())
+
+    res = pyperf.perf_counter() - t0
+
+    finalize(results)
+
+    return res
+
+
+if __name__ == '__main__':
+    runner = pyperf.Runner()
+    runner.bench_time_func('batch_append_v0', func, 0)
+    runner.bench_time_func('batch_append_v1', func, 1)
+    runner.bench_time_func('batch_append_v2', func, 2)

venv/lib/python3.11/site-packages/kafka/benchmarks/record_batch_read.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import hashlib
+import itertools
+import os
+import random
+
+import pyperf
+
+from kafka.record.memory_records import MemoryRecords, MemoryRecordsBuilder
+
+
+DEFAULT_BATCH_SIZE = 1600 * 1024
+KEY_SIZE = 6
+VALUE_SIZE = 60
+TIMESTAMP_RANGE = [1505824130000, 1505824140000]
+
+BATCH_SAMPLES = 5
+MESSAGES_PER_BATCH = 100
+
+
+def random_bytes(length):
+    buffer = bytearray(length)
+    for i in range(length):
+        buffer[i] = random.randint(0, 255)
+    return bytes(buffer)
+
+
+def prepare(magic):
+    samples = []
+    for _ in range(BATCH_SAMPLES):
+        batch = MemoryRecordsBuilder(
+            magic, batch_size=DEFAULT_BATCH_SIZE, compression_type=0)
+        for _ in range(MESSAGES_PER_BATCH):
+            size = batch.append(
+                random.randint(*TIMESTAMP_RANGE),
+                random_bytes(KEY_SIZE),
+                random_bytes(VALUE_SIZE),
+                headers=[])
+            assert size
+        batch.close()
+        samples.append(bytes(batch.buffer()))
+
+    return iter(itertools.cycle(samples))
+
+
+def finalize(results):
+    # Just some strange code to make sure PyPy does execute the code above
+    # properly
+    hash_val = hashlib.md5()
+    for buf in results:
+        hash_val.update(buf)
+    print(hash_val, file=open(os.devnull, "w"))
+
+
+def func(loops, magic):
+    # Jit can optimize out the whole function if the result is the same each
+    # time, so we need some randomized input data )
+    precomputed_samples = prepare(magic)
+    results = []
+
+    # Main benchmark code.
+    batch_data = next(precomputed_samples)
+    t0 = pyperf.perf_counter()
+    for _ in range(loops):
+        records = MemoryRecords(batch_data)
+        while records.has_next():
+            batch = records.next_batch()
+            batch.validate_crc()
+            for record in batch:
+                results.append(record.value)
+
+    res = pyperf.perf_counter() - t0
+    finalize(results)
+
+    return res
+
+
+if __name__ == '__main__':
+    runner = pyperf.Runner()
+    runner.bench_time_func('batch_read_v0', func, 0)
+    runner.bench_time_func('batch_read_v1', func, 1)
+    runner.bench_time_func('batch_read_v2', func, 2)

venv/lib/python3.11/site-packages/kafka/benchmarks/varint_speed.py

@@ -0,0 +1,434 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import pyperf
+from kafka.vendor import six
+
+
+test_data = [
+    (b"\x00", 0),
+    (b"\x01", -1),
+    (b"\x02", 1),
+    (b"\x7E", 63),
+    (b"\x7F", -64),
+    (b"\x80\x01", 64),
+    (b"\x81\x01", -65),
+    (b"\xFE\x7F", 8191),
+    (b"\xFF\x7F", -8192),
+    (b"\x80\x80\x01", 8192),
+    (b"\x81\x80\x01", -8193),
+    (b"\xFE\xFF\x7F", 1048575),
+    (b"\xFF\xFF\x7F", -1048576),
+    (b"\x80\x80\x80\x01", 1048576),
+    (b"\x81\x80\x80\x01", -1048577),
+    (b"\xFE\xFF\xFF\x7F", 134217727),
+    (b"\xFF\xFF\xFF\x7F", -134217728),
+    (b"\x80\x80\x80\x80\x01", 134217728),
+    (b"\x81\x80\x80\x80\x01", -134217729),
+    (b"\xFE\xFF\xFF\xFF\x7F", 17179869183),
+    (b"\xFF\xFF\xFF\xFF\x7F", -17179869184),
+    (b"\x80\x80\x80\x80\x80\x01", 17179869184),
+    (b"\x81\x80\x80\x80\x80\x01", -17179869185),
+    (b"\xFE\xFF\xFF\xFF\xFF\x7F", 2199023255551),
+    (b"\xFF\xFF\xFF\xFF\xFF\x7F", -2199023255552),
+    (b"\x80\x80\x80\x80\x80\x80\x01", 2199023255552),
+    (b"\x81\x80\x80\x80\x80\x80\x01", -2199023255553),
+    (b"\xFE\xFF\xFF\xFF\xFF\xFF\x7F", 281474976710655),
+    (b"\xFF\xFF\xFF\xFF\xFF\xFF\x7F", -281474976710656),
+    (b"\x80\x80\x80\x80\x80\x80\x80\x01", 281474976710656),
+    (b"\x81\x80\x80\x80\x80\x80\x80\x01", -281474976710657),
+    (b"\xFE\xFF\xFF\xFF\xFF\xFF\xFF\x7F", 36028797018963967),
+    (b"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F", -36028797018963968),
+    (b"\x80\x80\x80\x80\x80\x80\x80\x80\x01", 36028797018963968),
+    (b"\x81\x80\x80\x80\x80\x80\x80\x80\x01", -36028797018963969),
+    (b"\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F", 4611686018427387903),
+    (b"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F", -4611686018427387904),
+    (b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01", 4611686018427387904),
+    (b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x01", -4611686018427387905),
+]
+
+
+BENCH_VALUES_ENC = [
+    60,  # 1 byte
+    -8192,  # 2 bytes
+    1048575,  # 3 bytes
+    134217727,  # 4 bytes
+    -17179869184,  # 5 bytes
+    2199023255551,  # 6 bytes
+]
+
+BENCH_VALUES_DEC = [
+    b"\x7E",  # 1 byte
+    b"\xFF\x7F",  # 2 bytes
+    b"\xFE\xFF\x7F",  # 3 bytes
+    b"\xFF\xFF\xFF\x7F",  # 4 bytes
+    b"\x80\x80\x80\x80\x01",  # 5 bytes
+    b"\xFE\xFF\xFF\xFF\xFF\x7F",  # 6 bytes
+]
+BENCH_VALUES_DEC = list(map(bytearray, BENCH_VALUES_DEC))
+
+
+def _assert_valid_enc(enc_func):
+    for encoded, decoded in test_data:
+        assert enc_func(decoded) == encoded, decoded
+
+
+def _assert_valid_dec(dec_func):
+    for encoded, decoded in test_data:
+        res, pos = dec_func(bytearray(encoded))
+        assert res == decoded, (decoded, res)
+        assert pos == len(encoded), (decoded, pos)
+
+
+def _assert_valid_size(size_func):
+    for encoded, decoded in test_data:
+        assert size_func(decoded) == len(encoded), decoded
+
+
+def encode_varint_1(num):
+    """ Encode an integer to a varint presentation. See
+    https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints
+    on how those can be produced.
+
+        Arguments:
+            num (int): Value to encode
+
+        Returns:
+            bytearray: Encoded presentation of integer with length from 1 to 10
+                 bytes
+    """
+    # Shift sign to the end of number
+    num = (num << 1) ^ (num >> 63)
+    # Max 10 bytes. We assert those are allocated
+    buf = bytearray(10)
+
+    for i in range(10):
+        # 7 lowest bits from the number and set 8th if we still have pending
+        # bits left to encode
+        buf[i] = num & 0x7f | (0x80 if num > 0x7f else 0)
+        num = num >> 7
+        if num == 0:
+            break
+    else:
+        # Max size of endcoded double is 10 bytes for unsigned values
+        raise ValueError("Out of double range")
+    return buf[:i + 1]
+
+
+def encode_varint_2(value, int2byte=six.int2byte):
+    value = (value << 1) ^ (value >> 63)
+
+    bits = value & 0x7f
+    value >>= 7
+    res = b""
+    while value:
+        res += int2byte(0x80 | bits)
+        bits = value & 0x7f
+        value >>= 7
+    return res + int2byte(bits)
+
+
+def encode_varint_3(value, buf):
+    append = buf.append
+    value = (value << 1) ^ (value >> 63)
+
+    bits = value & 0x7f
+    value >>= 7
+    while value:
+        append(0x80 | bits)
+        bits = value & 0x7f
+        value >>= 7
+    append(bits)
+    return value
+
+
+def encode_varint_4(value, int2byte=six.int2byte):
+    value = (value << 1) ^ (value >> 63)
+
+    if value <= 0x7f:  # 1 byte
+        return int2byte(value)
+    if value <= 0x3fff:  # 2 bytes
+        return int2byte(0x80 | (value & 0x7f)) + int2byte(value >> 7)
+    if value <= 0x1fffff:  # 3 bytes
+        return int2byte(0x80 | (value & 0x7f)) + \
+            int2byte(0x80 | ((value >> 7) & 0x7f)) + \
+            int2byte(value >> 14)
+    if value <= 0xfffffff:  # 4 bytes
+        return int2byte(0x80 | (value & 0x7f)) + \
+            int2byte(0x80 | ((value >> 7) & 0x7f)) + \
+            int2byte(0x80 | ((value >> 14) & 0x7f)) + \
+            int2byte(value >> 21)
+    if value <= 0x7ffffffff:  # 5 bytes
+        return int2byte(0x80 | (value & 0x7f)) + \
+            int2byte(0x80 | ((value >> 7) & 0x7f)) + \
+            int2byte(0x80 | ((value >> 14) & 0x7f)) + \
+            int2byte(0x80 | ((value >> 21) & 0x7f)) + \
+            int2byte(value >> 28)
+    else:
+        # Return to general algorithm
+        bits = value & 0x7f
+        value >>= 7
+        res = b""
+        while value:
+            res += int2byte(0x80 | bits)
+            bits = value & 0x7f
+            value >>= 7
+        return res + int2byte(bits)
+
+
+def encode_varint_5(value, buf, pos=0):
+    value = (value << 1) ^ (value >> 63)
+
+    bits = value & 0x7f
+    value >>= 7
+    while value:
+        buf[pos] = 0x80 | bits
+        bits = value & 0x7f
+        value >>= 7
+        pos += 1
+    buf[pos] = bits
+    return pos + 1
+
+def encode_varint_6(value, buf):
+    append = buf.append
+    value = (value << 1) ^ (value >> 63)
+
+    if value <= 0x7f:  # 1 byte
+        append(value)
+        return 1
+    if value <= 0x3fff:  # 2 bytes
+        append(0x80 | (value & 0x7f))
+        append(value >> 7)
+        return 2
+    if value <= 0x1fffff:  # 3 bytes
+        append(0x80 | (value & 0x7f))
+        append(0x80 | ((value >> 7) & 0x7f))
+        append(value >> 14)
+        return 3
+    if value <= 0xfffffff:  # 4 bytes
+        append(0x80 | (value & 0x7f))
+        append(0x80 | ((value >> 7) & 0x7f))
+        append(0x80 | ((value >> 14) & 0x7f))
+        append(value >> 21)
+        return 4
+    if value <= 0x7ffffffff:  # 5 bytes
+        append(0x80 | (value & 0x7f))
+        append(0x80 | ((value >> 7) & 0x7f))
+        append(0x80 | ((value >> 14) & 0x7f))
+        append(0x80 | ((value >> 21) & 0x7f))
+        append(value >> 28)
+        return 5
+    else:
+        # Return to general algorithm
+        bits = value & 0x7f
+        value >>= 7
+        i = 0
+        while value:
+            append(0x80 | bits)
+            bits = value & 0x7f
+            value >>= 7
+            i += 1
+    append(bits)
+    return i
+
+
+def size_of_varint_1(value):
+    """ Number of bytes needed to encode an integer in variable-length format.
+    """
+    value = (value << 1) ^ (value >> 63)
+    res = 0
+    while True:
+        res += 1
+        value = value >> 7
+        if value == 0:
+            break
+    return res
+
+
+def size_of_varint_2(value):
+    """ Number of bytes needed to encode an integer in variable-length format.
+    """
+    value = (value << 1) ^ (value >> 63)
+    if value <= 0x7f:
+        return 1
+    if value <= 0x3fff:
+        return 2
+    if value <= 0x1fffff:
+        return 3
+    if value <= 0xfffffff:
+        return 4
+    if value <= 0x7ffffffff:
+        return 5
+    if value <= 0x3ffffffffff:
+        return 6
+    if value <= 0x1ffffffffffff:
+        return 7
+    if value <= 0xffffffffffffff:
+        return 8
+    if value <= 0x7fffffffffffffff:
+        return 9
+    return 10
+
+
+if six.PY3:
+    def _read_byte(memview, pos):
+        """ Read a byte from memoryview as an integer
+
+            Raises:
+                IndexError: if position is out of bounds
+        """
+        return memview[pos]
+else:
+    def _read_byte(memview, pos):
+        """ Read a byte from memoryview as an integer
+
+            Raises:
+                IndexError: if position is out of bounds
+        """
+        return ord(memview[pos])
+
+
+def decode_varint_1(buffer, pos=0):
+    """ Decode an integer from a varint presentation. See
+    https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints
+    on how those can be produced.
+
+        Arguments:
+            buffer (bytes-like): any object acceptable by ``memoryview``
+            pos (int): optional position to read from
+
+        Returns:
+            (int, int): Decoded int value and next read position
+    """
+    value = 0
+    shift = 0
+    memview = memoryview(buffer)
+    for i in range(pos, pos + 10):
+        try:
+            byte = _read_byte(memview, i)
+        except IndexError:
+            raise ValueError("End of byte stream")
+        if byte & 0x80 != 0:
+            value |= (byte & 0x7f) << shift
+            shift += 7
+        else:
+            value |= byte << shift
+            break
+    else:
+        # Max size of endcoded double is 10 bytes for unsigned values
+        raise ValueError("Out of double range")
+    # Normalize sign
+    return (value >> 1) ^ -(value & 1), i + 1
+
+
+def decode_varint_2(buffer, pos=0):
+    result = 0
+    shift = 0
+    while 1:
+        b = buffer[pos]
+        result |= ((b & 0x7f) << shift)
+        pos += 1
+        if not (b & 0x80):
+            # result = result_type(() & mask)
+            return ((result >> 1) ^ -(result & 1), pos)
+        shift += 7
+        if shift >= 64:
+            raise ValueError("Out of int64 range")
+
+
+def decode_varint_3(buffer, pos=0):
+    result = buffer[pos]
+    if not (result & 0x81):
+        return (result >> 1), pos + 1
+    if not (result & 0x80):
+        return (result >> 1) ^ (~0), pos + 1
+
+    result &= 0x7f
+    pos += 1
+    shift = 7
+    while 1:
+        b = buffer[pos]
+        result |= ((b & 0x7f) << shift)
+        pos += 1
+        if not (b & 0x80):
+            return ((result >> 1) ^ -(result & 1), pos)
+        shift += 7
+        if shift >= 64:
+            raise ValueError("Out of int64 range")
+
+
+if __name__ == '__main__':
+    _assert_valid_enc(encode_varint_1)
+    _assert_valid_enc(encode_varint_2)
+
+    for encoded, decoded in test_data:
+        res = bytearray()
+        encode_varint_3(decoded, res)
+        assert res == encoded
+
+    _assert_valid_enc(encode_varint_4)
+
+    # import dis
+    # dis.dis(encode_varint_4)
+
+    for encoded, decoded in test_data:
+        res = bytearray(10)
+        written = encode_varint_5(decoded, res)
+        assert res[:written] == encoded
+
+    for encoded, decoded in test_data:
+        res = bytearray()
+        encode_varint_6(decoded, res)
+        assert res == encoded
+
+    _assert_valid_size(size_of_varint_1)
+    _assert_valid_size(size_of_varint_2)
+    _assert_valid_dec(decode_varint_1)
+    _assert_valid_dec(decode_varint_2)
+    _assert_valid_dec(decode_varint_3)
+
+    # import dis
+    # dis.dis(decode_varint_3)
+
+    runner = pyperf.Runner()
+    # Encode algorithms returning a bytes result
+    for bench_func in [
+            encode_varint_1,
+            encode_varint_2,
+            encode_varint_4]:
+        for i, value in enumerate(BENCH_VALUES_ENC):
+            runner.bench_func(
+                '{}_{}byte'.format(bench_func.__name__, i + 1),
+                bench_func, value)
+
+    # Encode algorithms writing to the buffer
+    for bench_func in [
+            encode_varint_3,
+            encode_varint_5,
+            encode_varint_6]:
+        for i, value in enumerate(BENCH_VALUES_ENC):
+            fname = bench_func.__name__
+            runner.timeit(
+                '{}_{}byte'.format(fname, i + 1),
+                stmt="{}({}, buffer)".format(fname, value),
+                setup="from __main__ import {}; buffer = bytearray(10)".format(
+                    fname)
+            )
+
+    # Size algorithms
+    for bench_func in [
+            size_of_varint_1,
+            size_of_varint_2]:
+        for i, value in enumerate(BENCH_VALUES_ENC):
+            runner.bench_func(
+                '{}_{}byte'.format(bench_func.__name__, i + 1),
+                bench_func, value)
+
+    # Decode algorithms
+    for bench_func in [
+            decode_varint_1,
+            decode_varint_2,
+            decode_varint_3]:
+        for i, value in enumerate(BENCH_VALUES_DEC):
+            runner.bench_func(
+                '{}_{}byte'.format(bench_func.__name__, i + 1),
+                bench_func, value)

venv/lib/python3.11/site-packages/kafka/cluster.py

@@ -0,0 +1,448 @@
+from __future__ import absolute_import
+
+import collections
+import copy
+import logging
+import random
+import re
+import threading
+import time
+
+from kafka.vendor import six
+
+from kafka import errors as Errors
+from kafka.conn import get_ip_port_afi
+from kafka.future import Future
+from kafka.structs import BrokerMetadata, PartitionMetadata, TopicPartition
+
+log = logging.getLogger(__name__)
+
+
+class ClusterMetadata(object):
+    """
+    A class to manage kafka cluster metadata.
+
+    This class does not perform any IO. It simply updates internal state
+    given API responses (MetadataResponse, FindCoordinatorResponse).
+
+    Keyword Arguments:
+        retry_backoff_ms (int): Milliseconds to backoff when retrying on
+            errors. Default: 100.
+        metadata_max_age_ms (int): The period of time in milliseconds after
+            which we force a refresh of metadata even if we haven't seen any
+            partition leadership changes to proactively discover any new
+            brokers or partitions. Default: 300000
+        bootstrap_servers: 'host[:port]' string (or list of 'host[:port]'
+            strings) that the client should contact to bootstrap initial
+            cluster metadata. This does not have to be the full node list.
+            It just needs to have at least one broker that will respond to a
+            Metadata API Request. Default port is 9092. If no servers are
+            specified, will default to localhost:9092.
+    """
+    DEFAULT_CONFIG = {
+        'retry_backoff_ms': 100,
+        'metadata_max_age_ms': 300000,
+        'bootstrap_servers': [],
+    }
+
+    def __init__(self, **configs):
+        self._brokers = {}  # node_id -> BrokerMetadata
+        self._partitions = {}  # topic -> partition -> PartitionMetadata
+        self._broker_partitions = collections.defaultdict(set)  # node_id -> {TopicPartition...}
+        self._coordinators = {}  # (coord_type, coord_key) -> node_id
+        self._last_refresh_ms = 0
+        self._last_successful_refresh_ms = 0
+        self._need_update = True
+        self._future = None
+        self._listeners = set()
+        self._lock = threading.Lock()
+        self.need_all_topic_metadata = False
+        self.unauthorized_topics = set()
+        self.internal_topics = set()
+        self.controller = None
+        self.cluster_id = None
+
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        for key in self.config:
+            if key in configs:
+                self.config[key] = configs[key]
+
+        self._bootstrap_brokers = self._generate_bootstrap_brokers()
+        self._coordinator_brokers = {}
+
+    def _generate_bootstrap_brokers(self):
+        # collect_hosts does not perform DNS, so we should be fine to re-use
+        bootstrap_hosts = collect_hosts(self.config['bootstrap_servers'])
+
+        brokers = {}
+        for i, (host, port, _) in enumerate(bootstrap_hosts):
+            node_id = 'bootstrap-%s' % i
+            brokers[node_id] = BrokerMetadata(node_id, host, port, None)
+        return brokers
+
+    def is_bootstrap(self, node_id):
+        return node_id in self._bootstrap_brokers
+
+    def brokers(self):
+        """Get all BrokerMetadata
+
+        Returns:
+            set: {BrokerMetadata, ...}
+        """
+        return set(self._brokers.values()) or set(self._bootstrap_brokers.values())
+
+    def broker_metadata(self, broker_id):
+        """Get BrokerMetadata
+
+        Arguments:
+            broker_id (int or str): node_id for a broker to check
+
+        Returns:
+            BrokerMetadata or None if not found
+        """
+        return (
+            self._brokers.get(broker_id) or
+            self._bootstrap_brokers.get(broker_id) or
+            self._coordinator_brokers.get(broker_id)
+        )
+
+    def partitions_for_topic(self, topic):
+        """Return set of all partitions for topic (whether available or not)
+
+        Arguments:
+            topic (str): topic to check for partitions
+
+        Returns:
+            set: {partition (int), ...}
+            None if topic not found.
+        """
+        if topic not in self._partitions:
+            return None
+        return set(self._partitions[topic].keys())
+
+    def available_partitions_for_topic(self, topic):
+        """Return set of partitions with known leaders
+
+        Arguments:
+            topic (str): topic to check for partitions
+
+        Returns:
+            set: {partition (int), ...}
+            None if topic not found.
+        """
+        if topic not in self._partitions:
+            return None
+        return set([partition for partition, metadata
+                              in six.iteritems(self._partitions[topic])
+                              if metadata.leader != -1])
+
+    def leader_for_partition(self, partition):
+        """Return node_id of leader, -1 unavailable, None if unknown."""
+        if partition.topic not in self._partitions:
+            return None
+        elif partition.partition not in self._partitions[partition.topic]:
+            return None
+        return self._partitions[partition.topic][partition.partition].leader
+
+    def leader_epoch_for_partition(self, partition):
+        return self._partitions[partition.topic][partition.partition].leader_epoch
+
+    def partitions_for_broker(self, broker_id):
+        """Return TopicPartitions for which the broker is a leader.
+
+        Arguments:
+            broker_id (int or str): node id for a broker
+
+        Returns:
+            set: {TopicPartition, ...}
+            None if the broker either has no partitions or does not exist.
+        """
+        return self._broker_partitions.get(broker_id)
+
+    def coordinator_for_group(self, group):
+        """Return node_id of group coordinator.
+
+        Arguments:
+            group (str): name of consumer group
+
+        Returns:
+            node_id (int or str) for group coordinator, -1 if coordinator unknown
+            None if the group does not exist.
+        """
+        return self._coordinators.get(('group', group))
+
+    def ttl(self):
+        """Milliseconds until metadata should be refreshed"""
+        now = time.time() * 1000
+        if self._need_update:
+            ttl = 0
+        else:
+            metadata_age = now - self._last_successful_refresh_ms
+            ttl = self.config['metadata_max_age_ms'] - metadata_age
+
+        retry_age = now - self._last_refresh_ms
+        next_retry = self.config['retry_backoff_ms'] - retry_age
+
+        return max(ttl, next_retry, 0)
+
+    def refresh_backoff(self):
+        """Return milliseconds to wait before attempting to retry after failure"""
+        return self.config['retry_backoff_ms']
+
+    def request_update(self):
+        """Flags metadata for update, return Future()
+
+        Actual update must be handled separately. This method will only
+        change the reported ttl()
+
+        Returns:
+            kafka.future.Future (value will be the cluster object after update)
+        """
+        with self._lock:
+            self._need_update = True
+            if not self._future or self._future.is_done:
+                self._future = Future()
+            return self._future
+
+    @property
+    def need_update(self):
+        return self._need_update
+
+    def topics(self, exclude_internal_topics=True):
+        """Get set of known topics.
+
+        Arguments:
+            exclude_internal_topics (bool): Whether records from internal topics
+                (such as offsets) should be exposed to the consumer. If set to
+                True the only way to receive records from an internal topic is
+                subscribing to it. Default True
+
+        Returns:
+            set: {topic (str), ...}
+        """
+        topics = set(self._partitions.keys())
+        if exclude_internal_topics:
+            return topics - self.internal_topics
+        else:
+            return topics
+
+    def failed_update(self, exception):
+        """Update cluster state given a failed MetadataRequest."""
+        f = None
+        with self._lock:
+            if self._future:
+                f = self._future
+                self._future = None
+        if f:
+            f.failure(exception)
+        self._last_refresh_ms = time.time() * 1000
+
+    def update_metadata(self, metadata):
+        """Update cluster state given a MetadataResponse.
+
+        Arguments:
+            metadata (MetadataResponse): broker response to a metadata request
+
+        Returns: None
+        """
+        if not metadata.brokers:
+            log.warning("No broker metadata found in MetadataResponse -- ignoring.")
+            return self.failed_update(Errors.MetadataEmptyBrokerList(metadata))
+
+        _new_brokers = {}
+        for broker in metadata.brokers:
+            if metadata.API_VERSION == 0:
+                node_id, host, port = broker
+                rack = None
+            else:
+                node_id, host, port, rack = broker
+            _new_brokers.update({
+                node_id: BrokerMetadata(node_id, host, port, rack)
+            })
+
+        if metadata.API_VERSION == 0:
+            _new_controller = None
+        else:
+            _new_controller = _new_brokers.get(metadata.controller_id)
+
+        if metadata.API_VERSION < 2:
+            _new_cluster_id = None
+        else:
+            _new_cluster_id = metadata.cluster_id
+
+        _new_partitions = {}
+        _new_broker_partitions = collections.defaultdict(set)
+        _new_unauthorized_topics = set()
+        _new_internal_topics = set()
+
+        for topic_data in metadata.topics:
+            if metadata.API_VERSION == 0:
+                error_code, topic, partitions = topic_data
+                is_internal = False
+            else:
+                error_code, topic, is_internal, partitions = topic_data
+            if is_internal:
+                _new_internal_topics.add(topic)
+            error_type = Errors.for_code(error_code)
+            if error_type is Errors.NoError:
+                _new_partitions[topic] = {}
+                for partition_data in partitions:
+                    leader_epoch = -1
+                    offline_replicas = []
+                    if metadata.API_VERSION >= 7:
+                        p_error, partition, leader, leader_epoch, replicas, isr, offline_replicas = partition_data
+                    elif metadata.API_VERSION >= 5:
+                        p_error, partition, leader, replicas, isr, offline_replicas = partition_data
+                    else:
+                        p_error, partition, leader, replicas, isr = partition_data
+
+                    _new_partitions[topic][partition] = PartitionMetadata(
+                        topic=topic, partition=partition,
+                        leader=leader, leader_epoch=leader_epoch,
+                        replicas=replicas, isr=isr, offline_replicas=offline_replicas,
+                        error=p_error)
+                    if leader != -1:
+                        _new_broker_partitions[leader].add(
+                            TopicPartition(topic, partition))
+
+            # Specific topic errors can be ignored if this is a full metadata fetch
+            elif self.need_all_topic_metadata:
+                continue
+
+            elif error_type is Errors.LeaderNotAvailableError:
+                log.warning("Topic %s is not available during auto-create"
+                            " initialization", topic)
+            elif error_type is Errors.UnknownTopicOrPartitionError:
+                log.error("Topic %s not found in cluster metadata", topic)
+            elif error_type is Errors.TopicAuthorizationFailedError:
+                log.error("Topic %s is not authorized for this client", topic)
+                _new_unauthorized_topics.add(topic)
+            elif error_type is Errors.InvalidTopicError:
+                log.error("'%s' is not a valid topic name", topic)
+            else:
+                log.error("Error fetching metadata for topic %s: %s",
+                          topic, error_type)
+
+        with self._lock:
+            self._brokers = _new_brokers
+            self.controller = _new_controller
+            self.cluster_id = _new_cluster_id
+            self._partitions = _new_partitions
+            self._broker_partitions = _new_broker_partitions
+            self.unauthorized_topics = _new_unauthorized_topics
+            self.internal_topics = _new_internal_topics
+            f = None
+            if self._future:
+                f = self._future
+            self._future = None
+            self._need_update = False
+
+        now = time.time() * 1000
+        self._last_refresh_ms = now
+        self._last_successful_refresh_ms = now
+
+        if f:
+            # In the common case where we ask for a single topic and get back an
+            # error, we should fail the future
+            if len(metadata.topics) == 1 and metadata.topics[0][0] != Errors.NoError.errno:
+                error_code, topic = metadata.topics[0][:2]
+                error = Errors.for_code(error_code)(topic)
+                f.failure(error)
+            else:
+                f.success(self)
+
+        log.debug("Updated cluster metadata to %s", self)
+
+        for listener in self._listeners:
+            listener(self)
+
+        if self.need_all_topic_metadata:
+            # the listener may change the interested topics,
+            # which could cause another metadata refresh.
+            # If we have already fetched all topics, however,
+            # another fetch should be unnecessary.
+            self._need_update = False
+
+    def add_listener(self, listener):
+        """Add a callback function to be called on each metadata update"""
+        self._listeners.add(listener)
+
+    def remove_listener(self, listener):
+        """Remove a previously added listener callback"""
+        self._listeners.remove(listener)
+
+    def add_coordinator(self, response, coord_type, coord_key):
+        """Update with metadata for a group or txn coordinator
+
+        Arguments:
+            response (FindCoordinatorResponse): broker response
+            coord_type (str): 'group' or 'transaction'
+            coord_key (str): consumer_group or transactional_id
+
+        Returns:
+            string: coordinator node_id if metadata is updated, None on error
+        """
+        log.debug("Updating coordinator for %s/%s: %s", coord_type, coord_key, response)
+        error_type = Errors.for_code(response.error_code)
+        if error_type is not Errors.NoError:
+            log.error("FindCoordinatorResponse error: %s", error_type)
+            self._coordinators[(coord_type, coord_key)] = -1
+            return
+
+        # Use a coordinator-specific node id so that requests
+        # get a dedicated connection
+        node_id = 'coordinator-{}'.format(response.coordinator_id)
+        coordinator = BrokerMetadata(
+            node_id,
+            response.host,
+            response.port,
+            None)
+
+        log.info("Coordinator for %s/%s is %s", coord_type, coord_key, coordinator)
+        self._coordinator_brokers[node_id] = coordinator
+        self._coordinators[(coord_type, coord_key)] = node_id
+        return node_id
+
+    def with_partitions(self, partitions_to_add):
+        """Returns a copy of cluster metadata with partitions added"""
+        new_metadata = ClusterMetadata(**self.config)
+        new_metadata._brokers = copy.deepcopy(self._brokers)
+        new_metadata._partitions = copy.deepcopy(self._partitions)
+        new_metadata._broker_partitions = copy.deepcopy(self._broker_partitions)
+        new_metadata._coordinators = copy.deepcopy(self._coordinators)
+        new_metadata.internal_topics = copy.deepcopy(self.internal_topics)
+        new_metadata.unauthorized_topics = copy.deepcopy(self.unauthorized_topics)
+
+        for partition in partitions_to_add:
+            new_metadata._partitions[partition.topic][partition.partition] = partition
+
+            if partition.leader is not None and partition.leader != -1:
+                new_metadata._broker_partitions[partition.leader].add(
+                    TopicPartition(partition.topic, partition.partition))
+
+        return new_metadata
+
+    def __str__(self):
+        return 'ClusterMetadata(brokers: %d, topics: %d, coordinators: %d)' % \
+               (len(self._brokers), len(self._partitions), len(self._coordinators))
+
+
+def collect_hosts(hosts, randomize=True):
+    """
+    Collects a comma-separated set of hosts (host:port) and optionally
+    randomize the returned list.
+    """
+
+    if isinstance(hosts, six.string_types):
+        hosts = hosts.strip().split(',')
+
+    result = []
+    for host_port in hosts:
+        # ignore leading SECURITY_PROTOCOL:// to mimic java client
+        host_port = re.sub('^.*://', '', host_port)
+        host, port, afi = get_ip_port_afi(host_port)
+        result.append((host, port, afi))
+
+    if randomize:
+        random.shuffle(result)
+    return result

venv/lib/python3.11/site-packages/kafka/codec.py

@@ -0,0 +1,333 @@
+from __future__ import absolute_import
+
+import gzip
+import io
+import platform
+import struct
+
+from kafka.vendor import six
+from kafka.vendor.six.moves import range
+
+_XERIAL_V1_HEADER = (-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1)
+_XERIAL_V1_FORMAT = 'bccccccBii'
+ZSTD_MAX_OUTPUT_SIZE = 1024 * 1024
+
+try:
+    import snappy
+except ImportError:
+    snappy = None
+
+try:
+    import zstandard as zstd
+except ImportError:
+    zstd = None
+
+try:
+    import lz4.frame as lz4
+
+    def _lz4_compress(payload, **kwargs):
+        # Kafka does not support LZ4 dependent blocks
+        try:
+            # For lz4>=0.12.0
+            kwargs.pop('block_linked', None)
+            return lz4.compress(payload, block_linked=False, **kwargs)
+        except TypeError:
+            # For earlier versions of lz4
+            kwargs.pop('block_mode', None)
+            return lz4.compress(payload, block_mode=1, **kwargs)
+
+except ImportError:
+    lz4 = None
+
+try:
+    import lz4f
+except ImportError:
+    lz4f = None
+
+try:
+    import lz4framed
+except ImportError:
+    lz4framed = None
+
+try:
+    import xxhash
+except ImportError:
+    xxhash = None
+
+PYPY = bool(platform.python_implementation() == 'PyPy')
+
+def has_gzip():
+    return True
+
+
+def has_snappy():
+    return snappy is not None
+
+
+def has_zstd():
+    return zstd is not None
+
+
+def has_lz4():
+    if lz4 is not None:
+        return True
+    if lz4f is not None:
+        return True
+    if lz4framed is not None:
+        return True
+    return False
+
+
+def gzip_encode(payload, compresslevel=None):
+    if not compresslevel:
+        compresslevel = 9
+
+    buf = io.BytesIO()
+
+    # Gzip context manager introduced in python 2.7
+    # so old-fashioned way until we decide to not support 2.6
+    gzipper = gzip.GzipFile(fileobj=buf, mode="w", compresslevel=compresslevel)
+    try:
+        gzipper.write(payload)
+    finally:
+        gzipper.close()
+
+    return buf.getvalue()
+
+
+def gzip_decode(payload):
+    buf = io.BytesIO(payload)
+
+    # Gzip context manager introduced in python 2.7
+    # so old-fashioned way until we decide to not support 2.6
+    gzipper = gzip.GzipFile(fileobj=buf, mode='r')
+    try:
+        return gzipper.read()
+    finally:
+        gzipper.close()
+
+
+def snappy_encode(payload, xerial_compatible=True, xerial_blocksize=32*1024):
+    """Encodes the given data with snappy compression.
+
+    If xerial_compatible is set then the stream is encoded in a fashion
+    compatible with the xerial snappy library.
+
+    The block size (xerial_blocksize) controls how frequent the blocking occurs
+    32k is the default in the xerial library.
+
+    The format winds up being:
+
+
+        +-------------+------------+--------------+------------+--------------+
+        |   Header    | Block1 len | Block1 data  | Blockn len | Blockn data  |
+        +-------------+------------+--------------+------------+--------------+
+        |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
+        +-------------+------------+--------------+------------+--------------+
+
+
+    It is important to note that the blocksize is the amount of uncompressed
+    data presented to snappy at each block, whereas the blocklen is the number
+    of bytes that will be present in the stream; so the length will always be
+    <= blocksize.
+
+    """
+
+    if not has_snappy():
+        raise NotImplementedError("Snappy codec is not available")
+
+    if not xerial_compatible:
+        return snappy.compress(payload)
+
+    out = io.BytesIO()
+    for fmt, dat in zip(_XERIAL_V1_FORMAT, _XERIAL_V1_HEADER):
+        out.write(struct.pack('!' + fmt, dat))
+
+    # Chunk through buffers to avoid creating intermediate slice copies
+    if PYPY:
+        # on pypy, snappy.compress() on a sliced buffer consumes the entire
+        # buffer... likely a python-snappy bug, so just use a slice copy
+        chunker = lambda payload, i, size: payload[i:size+i]
+
+    elif six.PY2:
+        # Sliced buffer avoids additional copies
+        # pylint: disable-msg=undefined-variable
+        chunker = lambda payload, i, size: buffer(payload, i, size)
+    else:
+        # snappy.compress does not like raw memoryviews, so we have to convert
+        # tobytes, which is a copy... oh well. it's the thought that counts.
+        # pylint: disable-msg=undefined-variable
+        chunker = lambda payload, i, size: memoryview(payload)[i:size+i].tobytes()
+
+    for chunk in (chunker(payload, i, xerial_blocksize)
+                  for i in range(0, len(payload), xerial_blocksize)):
+
+        block = snappy.compress(chunk)
+        block_size = len(block)
+        out.write(struct.pack('!i', block_size))
+        out.write(block)
+
+    return out.getvalue()
+
+
+def _detect_xerial_stream(payload):
+    """Detects if the data given might have been encoded with the blocking mode
+        of the xerial snappy library.
+
+        This mode writes a magic header of the format:
+            +--------+--------------+------------+---------+--------+
+            | Marker | Magic String | Null / Pad | Version | Compat |
+            +--------+--------------+------------+---------+--------+
+            |  byte  |   c-string   |    byte    |  int32  | int32  |
+            +--------+--------------+------------+---------+--------+
+            |  -126  |   'SNAPPY'   |     \0     |         |        |
+            +--------+--------------+------------+---------+--------+
+
+        The pad appears to be to ensure that SNAPPY is a valid cstring
+        The version is the version of this format as written by xerial,
+        in the wild this is currently 1 as such we only support v1.
+
+        Compat is there to claim the minimum supported version that
+        can read a xerial block stream, presently in the wild this is
+        1.
+    """
+
+    if len(payload) > 16:
+        magic = struct.unpack('!' + _XERIAL_V1_FORMAT[:8], bytes(payload)[:8])
+        version, compat = struct.unpack('!' + _XERIAL_V1_FORMAT[8:], bytes(payload)[8:16])
+        # Until there is more than one way to do xerial blocking, the version + compat
+        # fields can be ignored. Also some producers (i.e., redpanda) are known to
+        # incorrectly encode these as little-endian, and that causes us to fail decoding
+        # when we otherwise would have succeeded.
+        # See https://github.com/dpkp/kafka-python/issues/2414
+        if magic == _XERIAL_V1_HEADER[:8]:
+            return True
+    return False
+
+
+def snappy_decode(payload):
+    if not has_snappy():
+        raise NotImplementedError("Snappy codec is not available")
+
+    if _detect_xerial_stream(payload):
+        # TODO ? Should become a fileobj ?
+        out = io.BytesIO()
+        byt = payload[16:]
+        length = len(byt)
+        cursor = 0
+
+        while cursor < length:
+            block_size = struct.unpack_from('!i', byt[cursor:])[0]
+            # Skip the block size
+            cursor += 4
+            end = cursor + block_size
+            out.write(snappy.decompress(byt[cursor:end]))
+            cursor = end
+
+        out.seek(0)
+        return out.read()
+    else:
+        return snappy.decompress(payload)
+
+
+if lz4:
+    lz4_encode = _lz4_compress # pylint: disable-msg=no-member
+elif lz4f:
+    lz4_encode = lz4f.compressFrame # pylint: disable-msg=no-member
+elif lz4framed:
+    lz4_encode = lz4framed.compress # pylint: disable-msg=no-member
+else:
+    lz4_encode = None
+
+
+def lz4f_decode(payload):
+    """Decode payload using interoperable LZ4 framing. Requires Kafka >= 0.10"""
+    # pylint: disable-msg=no-member
+    ctx = lz4f.createDecompContext()
+    data = lz4f.decompressFrame(payload, ctx)
+    lz4f.freeDecompContext(ctx)
+
+    # lz4f python module does not expose how much of the payload was
+    # actually read if the decompression was only partial.
+    if data['next'] != 0:
+        raise RuntimeError('lz4f unable to decompress full payload')
+    return data['decomp']
+
+
+if lz4:
+    lz4_decode = lz4.decompress # pylint: disable-msg=no-member
+elif lz4f:
+    lz4_decode = lz4f_decode
+elif lz4framed:
+    lz4_decode = lz4framed.decompress # pylint: disable-msg=no-member
+else:
+    lz4_decode = None
+
+
+def lz4_encode_old_kafka(payload):
+    """Encode payload for 0.8/0.9 brokers -- requires an incorrect header checksum."""
+    assert xxhash is not None
+    data = lz4_encode(payload)
+    header_size = 7
+    flg = data[4]
+    if not isinstance(flg, int):
+        flg = ord(flg)
+
+    content_size_bit = ((flg >> 3) & 1)
+    if content_size_bit:
+        # Old kafka does not accept the content-size field
+        # so we need to discard it and reset the header flag
+        flg -= 8
+        data = bytearray(data)
+        data[4] = flg
+        data = bytes(data)
+        payload = data[header_size+8:]
+    else:
+        payload = data[header_size:]
+
+    # This is the incorrect hc
+    hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member
+
+    return b''.join([
+        data[0:header_size-1],
+        hc,
+        payload
+    ])
+
+
+def lz4_decode_old_kafka(payload):
+    assert xxhash is not None
+    # Kafka's LZ4 code has a bug in its header checksum implementation
+    header_size = 7
+    if isinstance(payload[4], int):
+        flg = payload[4]
+    else:
+        flg = ord(payload[4])
+    content_size_bit = ((flg >> 3) & 1)
+    if content_size_bit:
+        header_size += 8
+
+    # This should be the correct hc
+    hc = xxhash.xxh32(payload[4:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member
+
+    munged_payload = b''.join([
+        payload[0:header_size-1],
+        hc,
+        payload[header_size:]
+    ])
+    return lz4_decode(munged_payload)
+
+
+def zstd_encode(payload):
+    if not zstd:
+        raise NotImplementedError("Zstd codec is not available")
+    return zstd.ZstdCompressor().compress(payload)
+
+
+def zstd_decode(payload):
+    if not zstd:
+        raise NotImplementedError("Zstd codec is not available")
+    try:
+        return zstd.ZstdDecompressor().decompress(payload)
+    except zstd.ZstdError:
+        return zstd.ZstdDecompressor().decompress(payload, max_output_size=ZSTD_MAX_OUTPUT_SIZE)

venv/lib/python3.11/site-packages/kafka/consumer/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import
+
+from kafka.consumer.group import KafkaConsumer
+
+__all__ = [
+    'KafkaConsumer'
+]

venv/lib/python3.11/site-packages/kafka/consumer/subscription_state.py

@@ -0,0 +1,589 @@
+from __future__ import absolute_import
+
+import abc
+from collections import OrderedDict
+try:
+    from collections.abc import Sequence
+except ImportError:
+    from collections import Sequence
+try:
+    # enum in stdlib as of py3.4
+    from enum import IntEnum  # pylint: disable=import-error
+except ImportError:
+    # vendored backport module
+    from kafka.vendor.enum34 import IntEnum
+import logging
+import random
+import re
+import threading
+import time
+
+from kafka.vendor import six
+
+import kafka.errors as Errors
+from kafka.protocol.list_offsets import OffsetResetStrategy
+from kafka.structs import OffsetAndMetadata
+from kafka.util import ensure_valid_topic_name, synchronized
+
+log = logging.getLogger(__name__)
+
+
+class SubscriptionType(IntEnum):
+    NONE = 0
+    AUTO_TOPICS = 1
+    AUTO_PATTERN = 2
+    USER_ASSIGNED = 3
+
+
+class SubscriptionState(object):
+    """
+    A class for tracking the topics, partitions, and offsets for the consumer.
+    A partition is "assigned" either directly with assign_from_user() (manual
+    assignment) or with assign_from_subscribed() (automatic assignment from
+    subscription).
+
+    Once assigned, the partition is not considered "fetchable" until its initial
+    position has been set with seek(). Fetchable partitions track a fetch
+    position which is used to set the offset of the next fetch, and a consumed
+    position which is the last offset that has been returned to the user. You
+    can suspend fetching from a partition through pause() without affecting the
+    fetched/consumed offsets. The partition will remain unfetchable until the
+    resume() is used. You can also query the pause state independently with
+    is_paused().
+
+    Note that pause state as well as fetch/consumed positions are not preserved
+    when partition assignment is changed whether directly by the user or
+    through a group rebalance.
+    """
+    _SUBSCRIPTION_EXCEPTION_MESSAGE = (
+        "You must choose only one way to configure your consumer:"
+        " (1) subscribe to specific topics by name,"
+        " (2) subscribe to topics matching a regex pattern,"
+        " (3) assign itself specific topic-partitions.")
+
+    def __init__(self, offset_reset_strategy='earliest'):
+        """Initialize a SubscriptionState instance
+
+        Keyword Arguments:
+            offset_reset_strategy: 'earliest' or 'latest', otherwise
+                exception will be raised when fetching an offset that is no
+                longer available. Default: 'earliest'
+        """
+        try:
+            offset_reset_strategy = getattr(OffsetResetStrategy,
+                                            offset_reset_strategy.upper())
+        except AttributeError:
+            log.warning('Unrecognized offset_reset_strategy, using NONE')
+            offset_reset_strategy = OffsetResetStrategy.NONE
+        self._default_offset_reset_strategy = offset_reset_strategy
+
+        self.subscription = None # set() or None
+        self.subscription_type = SubscriptionType.NONE
+        self.subscribed_pattern = None # regex str or None
+        self._group_subscription = set()
+        self._user_assignment = set()
+        self.assignment = OrderedDict()
+        self.rebalance_listener = None
+        self.listeners = []
+        self._lock = threading.RLock()
+
+    def _set_subscription_type(self, subscription_type):
+        if not isinstance(subscription_type, SubscriptionType):
+            raise ValueError('SubscriptionType enum required')
+        if self.subscription_type == SubscriptionType.NONE:
+            self.subscription_type = subscription_type
+        elif self.subscription_type != subscription_type:
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+
+    @synchronized
+    def subscribe(self, topics=(), pattern=None, listener=None):
+        """Subscribe to a list of topics, or a topic regex pattern.
+
+        Partitions will be dynamically assigned via a group coordinator.
+        Topic subscriptions are not incremental: this list will replace the
+        current assignment (if there is one).
+
+        This method is incompatible with assign_from_user()
+
+        Arguments:
+            topics (list): List of topics for subscription.
+            pattern (str): Pattern to match available topics. You must provide
+                either topics or pattern, but not both.
+            listener (ConsumerRebalanceListener): Optionally include listener
+                callback, which will be called before and after each rebalance
+                operation.
+
+                As part of group management, the consumer will keep track of the
+                list of consumers that belong to a particular group and will
+                trigger a rebalance operation if one of the following events
+                trigger:
+
+                * Number of partitions change for any of the subscribed topics
+                * Topic is created or deleted
+                * An existing member of the consumer group dies
+                * A new member is added to the consumer group
+
+                When any of these events are triggered, the provided listener
+                will be invoked first to indicate that the consumer's assignment
+                has been revoked, and then again when the new assignment has
+                been received. Note that this listener will immediately override
+                any listener set in a previous call to subscribe. It is
+                guaranteed, however, that the partitions revoked/assigned
+                through this interface are from topics subscribed in this call.
+        """
+        assert topics or pattern, 'Must provide topics or pattern'
+        if (topics and pattern):
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+
+        elif pattern:
+            self._set_subscription_type(SubscriptionType.AUTO_PATTERN)
+            log.info('Subscribing to pattern: /%s/', pattern)
+            self.subscription = set()
+            self.subscribed_pattern = re.compile(pattern)
+        else:
+            if isinstance(topics, str) or not isinstance(topics, Sequence):
+                raise TypeError('Topics must be a list (or non-str sequence)')
+            self._set_subscription_type(SubscriptionType.AUTO_TOPICS)
+            self.change_subscription(topics)
+
+        if listener and not isinstance(listener, ConsumerRebalanceListener):
+            raise TypeError('listener must be a ConsumerRebalanceListener')
+        self.rebalance_listener = listener
+
+    @synchronized
+    def change_subscription(self, topics):
+        """Change the topic subscription.
+
+        Arguments:
+            topics (list of str): topics for subscription
+
+        Raises:
+            IllegalStateError: if assign_from_user has been used already
+            TypeError: if a topic is None or a non-str
+            ValueError: if a topic is an empty string or
+                        - a topic name is '.' or '..' or
+                        - a topic name does not consist of ASCII-characters/'-'/'_'/'.'
+        """
+        if not self.partitions_auto_assigned():
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+
+        if isinstance(topics, six.string_types):
+            topics = [topics]
+
+        if self.subscription == set(topics):
+            log.warning("subscription unchanged by change_subscription(%s)",
+                        topics)
+            return
+
+        for t in topics:
+            ensure_valid_topic_name(t)
+
+        log.info('Updating subscribed topics to: %s', topics)
+        self.subscription = set(topics)
+        self._group_subscription.update(topics)
+
+    @synchronized
+    def group_subscribe(self, topics):
+        """Add topics to the current group subscription.
+
+        This is used by the group leader to ensure that it receives metadata
+        updates for all topics that any member of the group is subscribed to.
+
+        Arguments:
+            topics (list of str): topics to add to the group subscription
+        """
+        if not self.partitions_auto_assigned():
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+        self._group_subscription.update(topics)
+
+    @synchronized
+    def reset_group_subscription(self):
+        """Reset the group's subscription to only contain topics subscribed by this consumer."""
+        if not self.partitions_auto_assigned():
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+        assert self.subscription is not None, 'Subscription required'
+        self._group_subscription.intersection_update(self.subscription)
+
+    @synchronized
+    def assign_from_user(self, partitions):
+        """Manually assign a list of TopicPartitions to this consumer.
+
+        This interface does not allow for incremental assignment and will
+        replace the previous assignment (if there was one).
+
+        Manual topic assignment through this method does not use the consumer's
+        group management functionality. As such, there will be no rebalance
+        operation triggered when group membership or cluster and topic metadata
+        change. Note that it is not possible to use both manual partition
+        assignment with assign() and group assignment with subscribe().
+
+        Arguments:
+            partitions (list of TopicPartition): assignment for this instance.
+
+        Raises:
+            IllegalStateError: if consumer has already called subscribe()
+        """
+        self._set_subscription_type(SubscriptionType.USER_ASSIGNED)
+        if self._user_assignment != set(partitions):
+            self._user_assignment = set(partitions)
+            self._set_assignment({partition: self.assignment.get(partition, TopicPartitionState())
+                                  for partition in partitions})
+
+    @synchronized
+    def assign_from_subscribed(self, assignments):
+        """Update the assignment to the specified partitions
+
+        This method is called by the coordinator to dynamically assign
+        partitions based on the consumer's topic subscription. This is different
+        from assign_from_user() which directly sets the assignment from a
+        user-supplied TopicPartition list.
+
+        Arguments:
+            assignments (list of TopicPartition): partitions to assign to this
+                consumer instance.
+        """
+        if not self.partitions_auto_assigned():
+            raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
+
+        for tp in assignments:
+            if tp.topic not in self.subscription:
+                raise ValueError("Assigned partition %s for non-subscribed topic." % (tp,))
+
+        # randomized ordering should improve balance for short-lived consumers
+        self._set_assignment({partition: TopicPartitionState() for partition in assignments}, randomize=True)
+        log.info("Updated partition assignment: %s", assignments)
+
+    def _set_assignment(self, partition_states, randomize=False):
+        """Batch partition assignment by topic (self.assignment is OrderedDict)"""
+        self.assignment.clear()
+        topics = [tp.topic for tp in six.iterkeys(partition_states)]
+        if randomize:
+            random.shuffle(topics)
+        topic_partitions = OrderedDict({topic: [] for topic in topics})
+        for tp in six.iterkeys(partition_states):
+            topic_partitions[tp.topic].append(tp)
+        for topic in six.iterkeys(topic_partitions):
+            for tp in topic_partitions[topic]:
+                self.assignment[tp] = partition_states[tp]
+
+    @synchronized
+    def unsubscribe(self):
+        """Clear all topic subscriptions and partition assignments"""
+        self.subscription = None
+        self._user_assignment.clear()
+        self.assignment.clear()
+        self.subscribed_pattern = None
+        self.subscription_type = SubscriptionType.NONE
+
+    @synchronized
+    def group_subscription(self):
+        """Get the topic subscription for the group.
+
+        For the leader, this will include the union of all member subscriptions.
+        For followers, it is the member's subscription only.
+
+        This is used when querying topic metadata to detect metadata changes
+        that would require rebalancing (the leader fetches metadata for all
+        topics in the group so that it can do partition assignment).
+
+        Returns:
+            set: topics
+        """
+        return self._group_subscription
+
+    @synchronized
+    def seek(self, partition, offset):
+        """Manually specify the fetch offset for a TopicPartition.
+
+        Overrides the fetch offsets that the consumer will use on the next
+        poll(). If this API is invoked for the same partition more than once,
+        the latest offset will be used on the next poll(). Note that you may
+        lose data if this API is arbitrarily used in the middle of consumption,
+        to reset the fetch offsets.
+
+        Arguments:
+            partition (TopicPartition): partition for seek operation
+            offset (int or OffsetAndMetadata): message offset in partition
+        """
+        if not isinstance(offset, (int, OffsetAndMetadata)):
+            raise TypeError("offset must be type in or OffsetAndMetadata")
+        self.assignment[partition].seek(offset)
+
+    @synchronized
+    def assigned_partitions(self):
+        """Return set of TopicPartitions in current assignment."""
+        return set(self.assignment.keys())
+
+    @synchronized
+    def paused_partitions(self):
+        """Return current set of paused TopicPartitions."""
+        return set(partition for partition in self.assignment
+                   if self.is_paused(partition))
+
+    @synchronized
+    def fetchable_partitions(self):
+        """Return ordered list of TopicPartitions that should be Fetched."""
+        fetchable = list()
+        for partition, state in six.iteritems(self.assignment):
+            if state.is_fetchable():
+                fetchable.append(partition)
+        return fetchable
+
+    @synchronized
+    def partitions_auto_assigned(self):
+        """Return True unless user supplied partitions manually."""
+        return self.subscription_type in (SubscriptionType.AUTO_TOPICS, SubscriptionType.AUTO_PATTERN)
+
+    @synchronized
+    def all_consumed_offsets(self):
+        """Returns consumed offsets as {TopicPartition: OffsetAndMetadata}"""
+        all_consumed = {}
+        for partition, state in six.iteritems(self.assignment):
+            if state.has_valid_position:
+                all_consumed[partition] = state.position
+        return all_consumed
+
+    @synchronized
+    def request_offset_reset(self, partition, offset_reset_strategy=None):
+        """Mark partition for offset reset using specified or default strategy.
+
+        Arguments:
+            partition (TopicPartition): partition to mark
+            offset_reset_strategy (OffsetResetStrategy, optional)
+        """
+        if offset_reset_strategy is None:
+            offset_reset_strategy = self._default_offset_reset_strategy
+        self.assignment[partition].reset(offset_reset_strategy)
+
+    @synchronized
+    def set_reset_pending(self, partitions, next_allowed_reset_time):
+        for partition in partitions:
+            self.assignment[partition].set_reset_pending(next_allowed_reset_time)
+
+    @synchronized
+    def has_default_offset_reset_policy(self):
+        """Return True if default offset reset policy is Earliest or Latest"""
+        return self._default_offset_reset_strategy != OffsetResetStrategy.NONE
+
+    @synchronized
+    def is_offset_reset_needed(self, partition):
+        return self.assignment[partition].awaiting_reset
+
+    @synchronized
+    def has_all_fetch_positions(self):
+        for state in six.itervalues(self.assignment):
+            if not state.has_valid_position:
+                return False
+        return True
+
+    @synchronized
+    def missing_fetch_positions(self):
+        missing = set()
+        for partition, state in six.iteritems(self.assignment):
+            if state.is_missing_position():
+                missing.add(partition)
+        return missing
+
+    @synchronized
+    def has_valid_position(self, partition):
+        return partition in self.assignment and self.assignment[partition].has_valid_position
+
+    @synchronized
+    def reset_missing_positions(self):
+        partitions_with_no_offsets = set()
+        for tp, state in six.iteritems(self.assignment):
+            if state.is_missing_position():
+                if self._default_offset_reset_strategy == OffsetResetStrategy.NONE:
+                    partitions_with_no_offsets.add(tp)
+                else:
+                    state.reset(self._default_offset_reset_strategy)
+
+        if partitions_with_no_offsets:
+            raise Errors.NoOffsetForPartitionError(partitions_with_no_offsets)
+
+    @synchronized
+    def partitions_needing_reset(self):
+        partitions = set()
+        for tp, state in six.iteritems(self.assignment):
+            if state.awaiting_reset and state.is_reset_allowed():
+                partitions.add(tp)
+        return partitions
+
+    @synchronized
+    def is_assigned(self, partition):
+        return partition in self.assignment
+
+    @synchronized
+    def is_paused(self, partition):
+        return partition in self.assignment and self.assignment[partition].paused
+
+    @synchronized
+    def is_fetchable(self, partition):
+        return partition in self.assignment and self.assignment[partition].is_fetchable()
+
+    @synchronized
+    def pause(self, partition):
+        self.assignment[partition].pause()
+
+    @synchronized
+    def resume(self, partition):
+        self.assignment[partition].resume()
+
+    @synchronized
+    def reset_failed(self, partitions, next_retry_time):
+        for partition in partitions:
+            self.assignment[partition].reset_failed(next_retry_time)
+
+    @synchronized
+    def move_partition_to_end(self, partition):
+        if partition in self.assignment:
+            try:
+                self.assignment.move_to_end(partition)
+            except AttributeError:
+                state = self.assignment.pop(partition)
+                self.assignment[partition] = state
+
+    @synchronized
+    def position(self, partition):
+        return self.assignment[partition].position
+
+
+class TopicPartitionState(object):
+    def __init__(self):
+        self.paused = False # whether this partition has been paused by the user
+        self.reset_strategy = None # the reset strategy if awaiting_reset is set
+        self._position = None # OffsetAndMetadata exposed to the user
+        self.highwater = None
+        self.drop_pending_record_batch = False
+        self.next_allowed_retry_time = None
+
+    def _set_position(self, offset):
+        assert self.has_valid_position, 'Valid position required'
+        assert isinstance(offset, OffsetAndMetadata)
+        self._position = offset
+
+    def _get_position(self):
+        return self._position
+
+    position = property(_get_position, _set_position, None, "last position")
+
+    def reset(self, strategy):
+        assert strategy is not None
+        self.reset_strategy = strategy
+        self._position = None
+        self.next_allowed_retry_time = None
+
+    def is_reset_allowed(self):
+        return self.next_allowed_retry_time is None or self.next_allowed_retry_time < time.time()
+
+    @property
+    def awaiting_reset(self):
+        return self.reset_strategy is not None
+
+    def set_reset_pending(self, next_allowed_retry_time):
+        self.next_allowed_retry_time = next_allowed_retry_time
+
+    def reset_failed(self, next_allowed_retry_time):
+        self.next_allowed_retry_time = next_allowed_retry_time
+
+    @property
+    def has_valid_position(self):
+        return self._position is not None
+
+    def is_missing_position(self):
+        return not self.has_valid_position and not self.awaiting_reset
+
+    def seek(self, offset):
+        self._position = offset if isinstance(offset, OffsetAndMetadata) else OffsetAndMetadata(offset, '', -1)
+        self.reset_strategy = None
+        self.drop_pending_record_batch = True
+        self.next_allowed_retry_time = None
+
+    def pause(self):
+        self.paused = True
+
+    def resume(self):
+        self.paused = False
+
+    def is_fetchable(self):
+        return not self.paused and self.has_valid_position
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ConsumerRebalanceListener(object):
+    """
+    A callback interface that the user can implement to trigger custom actions
+    when the set of partitions assigned to the consumer changes.
+
+    This is applicable when the consumer is having Kafka auto-manage group
+    membership. If the consumer's directly assign partitions, those
+    partitions will never be reassigned and this callback is not applicable.
+
+    When Kafka is managing the group membership, a partition re-assignment will
+    be triggered any time the members of the group changes or the subscription
+    of the members changes. This can occur when processes die, new process
+    instances are added or old instances come back to life after failure.
+    Rebalances can also be triggered by changes affecting the subscribed
+    topics (e.g. when then number of partitions is administratively adjusted).
+
+    There are many uses for this functionality. One common use is saving offsets
+    in a custom store. By saving offsets in the on_partitions_revoked(), call we
+    can ensure that any time partition assignment changes the offset gets saved.
+
+    Another use is flushing out any kind of cache of intermediate results the
+    consumer may be keeping. For example, consider a case where the consumer is
+    subscribed to a topic containing user page views, and the goal is to count
+    the number of page views per users for each five minute window.  Let's say
+    the topic is partitioned by the user id so that all events for a particular
+    user will go to a single consumer instance. The consumer can keep in memory
+    a running tally of actions per user and only flush these out to a remote
+    data store when its cache gets too big. However if a partition is reassigned
+    it may want to automatically trigger a flush of this cache, before the new
+    owner takes over consumption.
+
+    This callback will execute in the user thread as part of the Consumer.poll()
+    whenever partition assignment changes.
+
+    It is guaranteed that all consumer processes will invoke
+    on_partitions_revoked() prior to any process invoking
+    on_partitions_assigned(). So if offsets or other state is saved in the
+    on_partitions_revoked() call, it should be saved by the time the process
+    taking over that partition has their on_partitions_assigned() callback
+    called to load the state.
+    """
+    @abc.abstractmethod
+    def on_partitions_revoked(self, revoked):
+        """
+        A callback method the user can implement to provide handling of offset
+        commits to a customized store on the start of a rebalance operation.
+        This method will be called before a rebalance operation starts and
+        after the consumer stops fetching data. It is recommended that offsets
+        should be committed in this callback to either Kafka or a custom offset
+        store to prevent duplicate data.
+
+        NOTE: This method is only called before rebalances. It is not called
+        prior to KafkaConsumer.close()
+
+        Arguments:
+            revoked (list of TopicPartition): the partitions that were assigned
+                to the consumer on the last rebalance
+        """
+        pass
+
+    @abc.abstractmethod
+    def on_partitions_assigned(self, assigned):
+        """
+        A callback method the user can implement to provide handling of
+        customized offsets on completion of a successful partition
+        re-assignment. This method will be called after an offset re-assignment
+        completes and before the consumer starts fetching data.
+
+        It is guaranteed that all the processes in a consumer group will execute
+        their on_partitions_revoked() callback before any instance executes its
+        on_partitions_assigned() callback.
+
+        Arguments:
+            assigned (list of TopicPartition): the partitions assigned to the
+                consumer (may include partitions that were previously assigned)
+        """
+        pass

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/abstract.py

@@ -0,0 +1,56 @@
+from __future__ import absolute_import
+
+import abc
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class AbstractPartitionAssignor(object):
+    """
+    Abstract assignor implementation which does some common grunt work (in particular collecting
+    partition counts which are always needed in assignors).
+    """
+
+    @abc.abstractproperty
+    def name(self):
+        """.name should be a string identifying the assignor"""
+        pass
+
+    @abc.abstractmethod
+    def assign(self, cluster, members):
+        """Perform group assignment given cluster metadata and member subscriptions
+
+        Arguments:
+            cluster (ClusterMetadata): metadata for use in assignment
+            members (dict of {member_id: MemberMetadata}): decoded metadata for
+                each member in the group.
+
+        Returns:
+            dict: {member_id: MemberAssignment}
+        """
+        pass
+
+    @abc.abstractmethod
+    def metadata(self, topics):
+        """Generate ProtocolMetadata to be submitted via JoinGroupRequest.
+
+        Arguments:
+            topics (set): a member's subscribed topics
+
+        Returns:
+            MemberMetadata struct
+        """
+        pass
+
+    @abc.abstractmethod
+    def on_assignment(self, assignment):
+        """Callback that runs on each assignment.
+
+        This method can be used to update internal state, if any, of the
+        partition assignor.
+
+        Arguments:
+            assignment (MemberAssignment): the member's assignment
+        """
+        pass

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/range.py

@@ -0,0 +1,77 @@
+from __future__ import absolute_import
+
+import collections
+import logging
+
+from kafka.vendor import six
+
+from kafka.coordinator.assignors.abstract import AbstractPartitionAssignor
+from kafka.coordinator.protocol import ConsumerProtocolMemberMetadata, ConsumerProtocolMemberAssignment
+
+log = logging.getLogger(__name__)
+
+
+class RangePartitionAssignor(AbstractPartitionAssignor):
+    """
+    The range assignor works on a per-topic basis. For each topic, we lay out
+    the available partitions in numeric order and the consumers in
+    lexicographic order. We then divide the number of partitions by the total
+    number of consumers to determine the number of partitions to assign to each
+    consumer. If it does not evenly divide, then the first few consumers will
+    have one extra partition.
+
+    For example, suppose there are two consumers C0 and C1, two topics t0 and
+    t1, and each topic has 3 partitions, resulting in partitions t0p0, t0p1,
+    t0p2, t1p0, t1p1, and t1p2.
+
+    The assignment will be:
+        C0: [t0p0, t0p1, t1p0, t1p1]
+        C1: [t0p2, t1p2]
+    """
+    name = 'range'
+    version = 0
+
+    @classmethod
+    def assign(cls, cluster, member_metadata):
+        consumers_per_topic = collections.defaultdict(list)
+        for member, metadata in six.iteritems(member_metadata):
+            for topic in metadata.subscription:
+                consumers_per_topic[topic].append(member)
+
+        # construct {member_id: {topic: [partition, ...]}}
+        assignment = collections.defaultdict(dict)
+
+        for topic, consumers_for_topic in six.iteritems(consumers_per_topic):
+            partitions = cluster.partitions_for_topic(topic)
+            if partitions is None:
+                log.warning('No partition metadata for topic %s', topic)
+                continue
+            partitions = sorted(partitions)
+            consumers_for_topic.sort()
+
+            partitions_per_consumer = len(partitions) // len(consumers_for_topic)
+            consumers_with_extra = len(partitions) % len(consumers_for_topic)
+
+            for i, member in enumerate(consumers_for_topic):
+                start = partitions_per_consumer * i
+                start += min(i, consumers_with_extra)
+                length = partitions_per_consumer
+                if not i + 1 > consumers_with_extra:
+                    length += 1
+                assignment[member][topic] = partitions[start:start+length]
+
+        protocol_assignment = {}
+        for member_id in member_metadata:
+            protocol_assignment[member_id] = ConsumerProtocolMemberAssignment(
+                cls.version,
+                sorted(assignment[member_id].items()),
+                b'')
+        return protocol_assignment
+
+    @classmethod
+    def metadata(cls, topics):
+        return ConsumerProtocolMemberMetadata(cls.version, list(topics), b'')
+
+    @classmethod
+    def on_assignment(cls, assignment):
+        pass

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/roundrobin.py

@@ -0,0 +1,96 @@
+from __future__ import absolute_import
+
+import collections
+import itertools
+import logging
+
+from kafka.vendor import six
+
+from kafka.coordinator.assignors.abstract import AbstractPartitionAssignor
+from kafka.coordinator.protocol import ConsumerProtocolMemberMetadata, ConsumerProtocolMemberAssignment
+from kafka.structs import TopicPartition
+
+log = logging.getLogger(__name__)
+
+
+class RoundRobinPartitionAssignor(AbstractPartitionAssignor):
+    """
+    The roundrobin assignor lays out all the available partitions and all the
+    available consumers. It then proceeds to do a roundrobin assignment from
+    partition to consumer. If the subscriptions of all consumer instances are
+    identical, then the partitions will be uniformly distributed. (i.e., the
+    partition ownership counts will be within a delta of exactly one across all
+    consumers.)
+
+    For example, suppose there are two consumers C0 and C1, two topics t0 and
+    t1, and each topic has 3 partitions, resulting in partitions t0p0, t0p1,
+    t0p2, t1p0, t1p1, and t1p2.
+
+    The assignment will be:
+        C0: [t0p0, t0p2, t1p1]
+        C1: [t0p1, t1p0, t1p2]
+
+    When subscriptions differ across consumer instances, the assignment process
+    still considers each consumer instance in round robin fashion but skips
+    over an instance if it is not subscribed to the topic. Unlike the case when
+    subscriptions are identical, this can result in imbalanced assignments.
+
+    For example, suppose we have three consumers C0, C1, C2, and three topics
+    t0, t1, t2, with unbalanced partitions t0p0, t1p0, t1p1, t2p0, t2p1, t2p2,
+    where C0 is subscribed to t0; C1 is subscribed to t0, t1; and C2 is
+    subscribed to t0, t1, t2.
+
+    The assignment will be:
+        C0: [t0p0]
+        C1: [t1p0]
+        C2: [t1p1, t2p0, t2p1, t2p2]
+    """
+    name = 'roundrobin'
+    version = 0
+
+    @classmethod
+    def assign(cls, cluster, member_metadata):
+        all_topics = set()
+        for metadata in six.itervalues(member_metadata):
+            all_topics.update(metadata.subscription)
+
+        all_topic_partitions = []
+        for topic in all_topics:
+            partitions = cluster.partitions_for_topic(topic)
+            if partitions is None:
+                log.warning('No partition metadata for topic %s', topic)
+                continue
+            for partition in partitions:
+                all_topic_partitions.append(TopicPartition(topic, partition))
+        all_topic_partitions.sort()
+
+        # construct {member_id: {topic: [partition, ...]}}
+        assignment = collections.defaultdict(lambda: collections.defaultdict(list))
+
+        member_iter = itertools.cycle(sorted(member_metadata.keys()))
+        for partition in all_topic_partitions:
+            member_id = next(member_iter)
+
+            # Because we constructed all_topic_partitions from the set of
+            # member subscribed topics, we should be safe assuming that
+            # each topic in all_topic_partitions is in at least one member
+            # subscription; otherwise this could yield an infinite loop
+            while partition.topic not in member_metadata[member_id].subscription:
+                member_id = next(member_iter)
+            assignment[member_id][partition.topic].append(partition.partition)
+
+        protocol_assignment = {}
+        for member_id in member_metadata:
+            protocol_assignment[member_id] = ConsumerProtocolMemberAssignment(
+                cls.version,
+                sorted(assignment[member_id].items()),
+                b'')
+        return protocol_assignment
+
+    @classmethod
+    def metadata(cls, topics):
+        return ConsumerProtocolMemberMetadata(cls.version, list(topics), b'')
+
+    @classmethod
+    def on_assignment(cls, assignment):
+        pass

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/sticky/partition_movements.py

@@ -0,0 +1,149 @@
+import logging
+from collections import defaultdict, namedtuple
+from copy import deepcopy
+
+from kafka.vendor import six
+
+log = logging.getLogger(__name__)
+
+
+ConsumerPair = namedtuple("ConsumerPair", ["src_member_id", "dst_member_id"])
+"""
+Represents a pair of Kafka consumer ids involved in a partition reassignment.
+Each ConsumerPair corresponds to a particular partition or topic, indicates that the particular partition or some
+partition of the particular topic was moved from the source consumer to the destination consumer
+during the rebalance. This class helps in determining whether a partition reassignment results in cycles among
+the generated graph of consumer pairs.
+"""
+
+
+def is_sublist(source, target):
+    """Checks if one list is a sublist of another.
+
+    Arguments:
+      source: the list in which to search for the occurrence of target.
+      target: the list to search for as a sublist of source
+
+    Returns:
+      true if target is in source; false otherwise
+    """
+    for index in (i for i, e in enumerate(source) if e == target[0]):
+        if tuple(source[index: index + len(target)]) == target:
+            return True
+    return False
+
+
+class PartitionMovements:
+    """
+    This class maintains some data structures to simplify lookup of partition movements among consumers.
+    At each point of time during a partition rebalance it keeps track of partition movements
+    corresponding to each topic, and also possible movement (in form a ConsumerPair object) for each partition.
+    """
+
+    def __init__(self):
+        self.partition_movements_by_topic = defaultdict(
+            lambda: defaultdict(set)
+        )
+        self.partition_movements = {}
+
+    def move_partition(self, partition, old_consumer, new_consumer):
+        pair = ConsumerPair(src_member_id=old_consumer, dst_member_id=new_consumer)
+        if partition in self.partition_movements:
+            # this partition has previously moved
+            existing_pair = self._remove_movement_record_of_partition(partition)
+            assert existing_pair.dst_member_id == old_consumer
+            if existing_pair.src_member_id != new_consumer:
+                # the partition is not moving back to its previous consumer
+                self._add_partition_movement_record(
+                    partition, ConsumerPair(src_member_id=existing_pair.src_member_id, dst_member_id=new_consumer)
+                )
+        else:
+            self._add_partition_movement_record(partition, pair)
+
+    def get_partition_to_be_moved(self, partition, old_consumer, new_consumer):
+        if partition.topic not in self.partition_movements_by_topic:
+            return partition
+        if partition in self.partition_movements:
+            # this partition has previously moved
+            assert old_consumer == self.partition_movements[partition].dst_member_id
+            old_consumer = self.partition_movements[partition].src_member_id
+        reverse_pair = ConsumerPair(src_member_id=new_consumer, dst_member_id=old_consumer)
+        if reverse_pair not in self.partition_movements_by_topic[partition.topic]:
+            return partition
+
+        return next(iter(self.partition_movements_by_topic[partition.topic][reverse_pair]))
+
+    def are_sticky(self):
+        for topic, movements in six.iteritems(self.partition_movements_by_topic):
+            movement_pairs = set(movements.keys())
+            if self._has_cycles(movement_pairs):
+                log.error(
+                    "Stickiness is violated for topic {}\n"
+                    "Partition movements for this topic occurred among the following consumer pairs:\n"
+                    "{}".format(topic, movement_pairs)
+                )
+                return False
+        return True
+
+    def _remove_movement_record_of_partition(self, partition):
+        pair = self.partition_movements[partition]
+        del self.partition_movements[partition]
+
+        self.partition_movements_by_topic[partition.topic][pair].remove(partition)
+        if not self.partition_movements_by_topic[partition.topic][pair]:
+            del self.partition_movements_by_topic[partition.topic][pair]
+        if not self.partition_movements_by_topic[partition.topic]:
+            del self.partition_movements_by_topic[partition.topic]
+
+        return pair
+
+    def _add_partition_movement_record(self, partition, pair):
+        self.partition_movements[partition] = pair
+        self.partition_movements_by_topic[partition.topic][pair].add(partition)
+
+    def _has_cycles(self, consumer_pairs):
+        cycles = set()
+        for pair in consumer_pairs:
+            reduced_pairs = deepcopy(consumer_pairs)
+            reduced_pairs.remove(pair)
+            path = [pair.src_member_id]
+            if self._is_linked(pair.dst_member_id, pair.src_member_id, reduced_pairs, path) and not self._is_subcycle(
+                path, cycles
+            ):
+                cycles.add(tuple(path))
+                log.error("A cycle of length {} was found: {}".format(len(path) - 1, path))
+
+        # for now we want to make sure there is no partition movements of the same topic between a pair of consumers.
+        # the odds of finding a cycle among more than two consumers seem to be very low (according to various randomized
+        # tests with the given sticky algorithm) that it should not worth the added complexity of handling those cases.
+        for cycle in cycles:
+            if len(cycle) == 3:  # indicates a cycle of length 2
+                return True
+        return False
+
+    @staticmethod
+    def _is_subcycle(cycle, cycles):
+        super_cycle = deepcopy(cycle)
+        super_cycle = super_cycle[:-1]
+        super_cycle.extend(cycle)
+        for found_cycle in cycles:
+            if len(found_cycle) == len(cycle) and is_sublist(super_cycle, found_cycle):
+                return True
+        return False
+
+    def _is_linked(self, src, dst, pairs, current_path):
+        if src == dst:
+            return False
+        if not pairs:
+            return False
+        if ConsumerPair(src, dst) in pairs:
+            current_path.append(src)
+            current_path.append(dst)
+            return True
+        for pair in pairs:
+            if pair.src_member_id == src:
+                reduced_set = deepcopy(pairs)
+                reduced_set.remove(pair)
+                current_path.append(pair.src_member_id)
+                return self._is_linked(pair.dst_member_id, dst, reduced_set, current_path)
+        return False

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/sticky/sorted_set.py

@@ -0,0 +1,63 @@
+class SortedSet:
+    def __init__(self, iterable=None, key=None):
+        self._key = key if key is not None else lambda x: x
+        self._set = set(iterable) if iterable is not None else set()
+
+        self._cached_last = None
+        self._cached_first = None
+
+    def first(self):
+        if self._cached_first is not None:
+            return self._cached_first
+
+        first = None
+        for element in self._set:
+            if first is None or self._key(first) > self._key(element):
+                first = element
+        self._cached_first = first
+        return first
+
+    def last(self):
+        if self._cached_last is not None:
+            return self._cached_last
+
+        last = None
+        for element in self._set:
+            if last is None or self._key(last) < self._key(element):
+                last = element
+        self._cached_last = last
+        return last
+
+    def pop_last(self):
+        value = self.last()
+        self._set.remove(value)
+        self._cached_last = None
+        return value
+
+    def add(self, value):
+        if self._cached_last is not None and self._key(value) > self._key(self._cached_last):
+            self._cached_last = value
+        if self._cached_first is not None and self._key(value) < self._key(self._cached_first):
+            self._cached_first = value
+
+        return self._set.add(value)
+
+    def remove(self, value):
+        if self._cached_last is not None and self._cached_last == value:
+            self._cached_last = None
+        if self._cached_first is not None and self._cached_first == value:
+            self._cached_first = None
+
+        return self._set.remove(value)
+
+    def __contains__(self, value):
+        return value in self._set
+
+    def __iter__(self):
+        return iter(sorted(self._set, key=self._key))
+
+    def _bool(self):
+        return len(self._set) != 0
+
+    __nonzero__ = _bool
+    __bool__ = _bool

venv/lib/python3.11/site-packages/kafka/coordinator/assignors/sticky/sticky_assignor.py

@@ -0,0 +1,684 @@
+import logging
+from collections import defaultdict, namedtuple
+from copy import deepcopy
+
+from kafka.coordinator.assignors.abstract import AbstractPartitionAssignor
+from kafka.coordinator.assignors.sticky.partition_movements import PartitionMovements
+from kafka.coordinator.assignors.sticky.sorted_set import SortedSet
+from kafka.coordinator.protocol import ConsumerProtocolMemberMetadata, ConsumerProtocolMemberAssignment
+from kafka.coordinator.protocol import Schema
+from kafka.protocol.struct import Struct
+from kafka.protocol.types import String, Array, Int32
+from kafka.structs import TopicPartition
+from kafka.vendor import six
+
+log = logging.getLogger(__name__)
+
+ConsumerGenerationPair = namedtuple("ConsumerGenerationPair", ["consumer", "generation"])
+
+
+def has_identical_list_elements(list_):
+    """Checks if all lists in the collection have the same members
+
+    Arguments:
+      list_: collection of lists
+
+    Returns:
+      true if all lists in the collection have the same members; false otherwise
+    """
+    if not list_:
+        return True
+    for i in range(1, len(list_)):
+        if list_[i] != list_[i - 1]:
+            return False
+    return True
+
+
+def subscriptions_comparator_key(element):
+    return len(element[1]), element[0]
+
+
+def partitions_comparator_key(element):
+    return len(element[1]), element[0].topic, element[0].partition
+
+
+def remove_if_present(collection, element):
+    try:
+        collection.remove(element)
+    except (ValueError, KeyError):
+        pass
+
+
+StickyAssignorMemberMetadataV1 = namedtuple("StickyAssignorMemberMetadataV1",
+                                            ["subscription", "partitions", "generation"])
+
+
+class StickyAssignorUserDataV1(Struct):
+    """
+    Used for preserving consumer's previously assigned partitions
+    list and sending it as user data to the leader during a rebalance
+    """
+
+    SCHEMA = Schema(
+        ("previous_assignment", Array(("topic", String("utf-8")), ("partitions", Array(Int32)))), ("generation", Int32)
+    )
+
+
+class StickyAssignmentExecutor:
+    def __init__(self, cluster, members):
+        self.members = members
+        # a mapping between consumers and their assigned partitions that is updated during assignment procedure
+        self.current_assignment = defaultdict(list)
+        # an assignment from a previous generation
+        self.previous_assignment = {}
+        # a mapping between partitions and their assigned consumers
+        self.current_partition_consumer = {}
+        # a flag indicating that there were no previous assignments performed ever
+        self.is_fresh_assignment = False
+        # a mapping of all topic partitions to all consumers that can be assigned to them
+        self.partition_to_all_potential_consumers = {}
+        # a mapping of all consumers to all potential topic partitions that can be assigned to them
+        self.consumer_to_all_potential_partitions = {}
+        # an ascending sorted set of consumers based on how many topic partitions are already assigned to them
+        self.sorted_current_subscriptions = SortedSet()
+        # an ascending sorted list of topic partitions based on how many consumers can potentially use them
+        self.sorted_partitions = []
+        # all partitions that need to be assigned
+        self.unassigned_partitions = []
+        # a flag indicating that a certain partition cannot remain assigned to its current consumer because the consumer
+        # is no longer subscribed to its topic
+        self.revocation_required = False
+
+        self.partition_movements = PartitionMovements()
+        self._initialize(cluster)
+
+    def perform_initial_assignment(self):
+        self._populate_sorted_partitions()
+        self._populate_partitions_to_reassign()
+
+    def balance(self):
+        self._initialize_current_subscriptions()
+        initializing = len(self.current_assignment[self._get_consumer_with_most_subscriptions()]) == 0
+
+        # assign all unassigned partitions
+        for partition in self.unassigned_partitions:
+            # skip if there is no potential consumer for the partition
+            if not self.partition_to_all_potential_consumers[partition]:
+                continue
+            self._assign_partition(partition)
+
+        # narrow down the reassignment scope to only those partitions that can actually be reassigned
+        fixed_partitions = set()
+        for partition in six.iterkeys(self.partition_to_all_potential_consumers):
+            if not self._can_partition_participate_in_reassignment(partition):
+                fixed_partitions.add(partition)
+        for fixed_partition in fixed_partitions:
+            remove_if_present(self.sorted_partitions, fixed_partition)
+            remove_if_present(self.unassigned_partitions, fixed_partition)
+
+        # narrow down the reassignment scope to only those consumers that are subject to reassignment
+        fixed_assignments = {}
+        for consumer in six.iterkeys(self.consumer_to_all_potential_partitions):
+            if not self._can_consumer_participate_in_reassignment(consumer):
+                self._remove_consumer_from_current_subscriptions_and_maintain_order(consumer)
+                fixed_assignments[consumer] = self.current_assignment[consumer]
+                del self.current_assignment[consumer]
+
+        # create a deep copy of the current assignment so we can revert to it
+        # if we do not get a more balanced assignment later
+        prebalance_assignment = deepcopy(self.current_assignment)
+        prebalance_partition_consumers = deepcopy(self.current_partition_consumer)
+
+        # if we don't already need to revoke something due to subscription changes,
+        # first try to balance by only moving newly added partitions
+        if not self.revocation_required:
+            self._perform_reassignments(self.unassigned_partitions)
+        reassignment_performed = self._perform_reassignments(self.sorted_partitions)
+
+        # if we are not preserving existing assignments and we have made changes to the current assignment
+        # make sure we are getting a more balanced assignment; otherwise, revert to previous assignment
+        if (
+            not initializing
+            and reassignment_performed
+            and self._get_balance_score(self.current_assignment) >= self._get_balance_score(prebalance_assignment)
+        ):
+            self.current_assignment = prebalance_assignment
+            self.current_partition_consumer.clear()
+            self.current_partition_consumer.update(prebalance_partition_consumers)
+
+        # add the fixed assignments (those that could not change) back
+        for consumer, partitions in six.iteritems(fixed_assignments):
+            self.current_assignment[consumer] = partitions
+            self._add_consumer_to_current_subscriptions_and_maintain_order(consumer)
+
+    def get_final_assignment(self, member_id):
+        assignment = defaultdict(list)
+        for topic_partition in self.current_assignment[member_id]:
+            assignment[topic_partition.topic].append(topic_partition.partition)
+        assignment = {k: sorted(v) for k, v in six.iteritems(assignment)}
+        return six.viewitems(assignment)
+
+    def _initialize(self, cluster):
+        self._init_current_assignments(self.members)
+
+        for topic in cluster.topics():
+            partitions = cluster.partitions_for_topic(topic)
+            if partitions is None:
+                log.warning("No partition metadata for topic %s", topic)
+                continue
+            for p in partitions:
+                partition = TopicPartition(topic=topic, partition=p)
+                self.partition_to_all_potential_consumers[partition] = []
+        for consumer_id, member_metadata in six.iteritems(self.members):
+            self.consumer_to_all_potential_partitions[consumer_id] = []
+            for topic in member_metadata.subscription:
+                if cluster.partitions_for_topic(topic) is None:
+                    log.warning("No partition metadata for topic {}".format(topic))
+                    continue
+                for p in cluster.partitions_for_topic(topic):
+                    partition = TopicPartition(topic=topic, partition=p)
+                    self.consumer_to_all_potential_partitions[consumer_id].append(partition)
+                    self.partition_to_all_potential_consumers[partition].append(consumer_id)
+            if consumer_id not in self.current_assignment:
+                self.current_assignment[consumer_id] = []
+
+    def _init_current_assignments(self, members):
+        # we need to process subscriptions' user data with each consumer's reported generation in mind
+        # higher generations overwrite lower generations in case of a conflict
+        # note that a conflict could exists only if user data is for different generations
+
+        # for each partition we create a map of its consumers by generation
+        sorted_partition_consumers_by_generation = {}
+        for consumer, member_metadata in six.iteritems(members):
+            for partitions in member_metadata.partitions:
+                if partitions in sorted_partition_consumers_by_generation:
+                    consumers = sorted_partition_consumers_by_generation[partitions]
+                    if member_metadata.generation and member_metadata.generation in consumers:
+                        # same partition is assigned to two consumers during the same rebalance.
+                        # log a warning and skip this record
+                        log.warning(
+                            "Partition {} is assigned to multiple consumers "
+                            "following sticky assignment generation {}.".format(partitions, member_metadata.generation)
+                        )
+                    else:
+                        consumers[member_metadata.generation] = consumer
+                else:
+                    sorted_consumers = {member_metadata.generation: consumer}
+                    sorted_partition_consumers_by_generation[partitions] = sorted_consumers
+
+        # previous_assignment holds the prior ConsumerGenerationPair (before current) of each partition
+        # current and previous consumers are the last two consumers of each partition in the above sorted map
+        for partitions, consumers in six.iteritems(sorted_partition_consumers_by_generation):
+            generations = sorted(consumers.keys(), reverse=True)
+            self.current_assignment[consumers[generations[0]]].append(partitions)
+            # now update previous assignment if any
+            if len(generations) > 1:
+                self.previous_assignment[partitions] = ConsumerGenerationPair(
+                    consumer=consumers[generations[1]], generation=generations[1]
+                )
+
+        self.is_fresh_assignment = len(self.current_assignment) == 0
+
+        for consumer_id, partitions in six.iteritems(self.current_assignment):
+            for partition in partitions:
+                self.current_partition_consumer[partition] = consumer_id
+
+    def _are_subscriptions_identical(self):
+        """
+        Returns:
+            true, if both potential consumers of partitions and potential partitions that consumers can
+            consume are the same
+        """
+        if not has_identical_list_elements(list(six.itervalues(self.partition_to_all_potential_consumers))):
+            return False
+        return has_identical_list_elements(list(six.itervalues(self.consumer_to_all_potential_partitions)))
+
+    def _populate_sorted_partitions(self):
+        # set of topic partitions with their respective potential consumers
+        all_partitions = set((tp, tuple(consumers))
+                             for tp, consumers in six.iteritems(self.partition_to_all_potential_consumers))
+        partitions_sorted_by_num_of_potential_consumers = sorted(all_partitions, key=partitions_comparator_key)
+
+        self.sorted_partitions = []
+        if not self.is_fresh_assignment and self._are_subscriptions_identical():
+            # if this is a reassignment and the subscriptions are identical (all consumers can consumer from all topics)
+            # then we just need to simply list partitions in a round robin fashion (from consumers with
+            # most assigned partitions to those with least)
+            assignments = deepcopy(self.current_assignment)
+            for consumer_id, partitions in six.iteritems(assignments):
+                to_remove = []
+                for partition in partitions:
+                    if partition not in self.partition_to_all_potential_consumers:
+                        to_remove.append(partition)
+                for partition in to_remove:
+                    partitions.remove(partition)
+
+            sorted_consumers = SortedSet(
+                iterable=[(consumer, tuple(partitions)) for consumer, partitions in six.iteritems(assignments)],
+                key=subscriptions_comparator_key,
+            )
+            # at this point, sorted_consumers contains an ascending-sorted list of consumers based on
+            # how many valid partitions are currently assigned to them
+            while sorted_consumers:
+                # take the consumer with the most partitions
+                consumer, _ = sorted_consumers.pop_last()
+                # currently assigned partitions to this consumer
+                remaining_partitions = assignments[consumer]
+                # from partitions that had a different consumer before,
+                # keep only those that are assigned to this consumer now
+                previous_partitions = set(six.iterkeys(self.previous_assignment)).intersection(set(remaining_partitions))
+                if previous_partitions:
+                    # if there is a partition of this consumer that was assigned to another consumer before
+                    # mark it as good options for reassignment
+                    partition = previous_partitions.pop()
+                    remaining_partitions.remove(partition)
+                    self.sorted_partitions.append(partition)
+                    sorted_consumers.add((consumer, tuple(assignments[consumer])))
+                elif remaining_partitions:
+                    # otherwise, mark any other one of the current partitions as a reassignment candidate
+                    self.sorted_partitions.append(remaining_partitions.pop())
+                    sorted_consumers.add((consumer, tuple(assignments[consumer])))
+
+            while partitions_sorted_by_num_of_potential_consumers:
+                partition = partitions_sorted_by_num_of_potential_consumers.pop(0)[0]
+                if partition not in self.sorted_partitions:
+                    self.sorted_partitions.append(partition)
+        else:
+            while partitions_sorted_by_num_of_potential_consumers:
+                self.sorted_partitions.append(partitions_sorted_by_num_of_potential_consumers.pop(0)[0])
+
+    def _populate_partitions_to_reassign(self):
+        self.unassigned_partitions = deepcopy(self.sorted_partitions)
+
+        assignments_to_remove = []
+        for consumer_id, partitions in six.iteritems(self.current_assignment):
+            if consumer_id not in self.members:
+                # if a consumer that existed before (and had some partition assignments) is now removed,
+                # remove it from current_assignment
+                for partition in partitions:
+                    del self.current_partition_consumer[partition]
+                assignments_to_remove.append(consumer_id)
+            else:
+                # otherwise (the consumer still exists)
+                partitions_to_remove = []
+                for partition in partitions:
+                    if partition not in self.partition_to_all_potential_consumers:
+                        # if this topic partition of this consumer no longer exists
+                        # remove it from current_assignment of the consumer
+                        partitions_to_remove.append(partition)
+                    elif partition.topic not in self.members[consumer_id].subscription:
+                        # if this partition cannot remain assigned to its current consumer because the consumer
+                        # is no longer subscribed to its topic remove it from current_assignment of the consumer
+                        partitions_to_remove.append(partition)
+                        self.revocation_required = True
+                    else:
+                        # otherwise, remove the topic partition from those that need to be assigned only if
+                        # its current consumer is still subscribed to its topic (because it is already assigned
+                        # and we would want to preserve that assignment as much as possible)
+                        self.unassigned_partitions.remove(partition)
+                for partition in partitions_to_remove:
+                    self.current_assignment[consumer_id].remove(partition)
+                    del self.current_partition_consumer[partition]
+        for consumer_id in assignments_to_remove:
+            del self.current_assignment[consumer_id]
+
+    def _initialize_current_subscriptions(self):
+        self.sorted_current_subscriptions = SortedSet(
+            iterable=[(consumer, tuple(partitions)) for consumer, partitions in six.iteritems(self.current_assignment)],
+            key=subscriptions_comparator_key,
+        )
+
+    def _get_consumer_with_least_subscriptions(self):
+        return self.sorted_current_subscriptions.first()[0]
+
+    def _get_consumer_with_most_subscriptions(self):
+        return self.sorted_current_subscriptions.last()[0]
+
+    def _remove_consumer_from_current_subscriptions_and_maintain_order(self, consumer):
+        self.sorted_current_subscriptions.remove((consumer, tuple(self.current_assignment[consumer])))
+
+    def _add_consumer_to_current_subscriptions_and_maintain_order(self, consumer):
+        self.sorted_current_subscriptions.add((consumer, tuple(self.current_assignment[consumer])))
+
+    def _is_balanced(self):
+        """Determines if the current assignment is a balanced one"""
+        if (
+            len(self.current_assignment[self._get_consumer_with_least_subscriptions()])
+            >= len(self.current_assignment[self._get_consumer_with_most_subscriptions()]) - 1
+        ):
+            # if minimum and maximum numbers of partitions assigned to consumers differ by at most one return true
+            return True
+
+        # create a mapping from partitions to the consumer assigned to them
+        all_assigned_partitions = {}
+        for consumer_id, consumer_partitions in six.iteritems(self.current_assignment):
+            for partition in consumer_partitions:
+                if partition in all_assigned_partitions:
+                    log.error("{} is assigned to more than one consumer.".format(partition))
+                all_assigned_partitions[partition] = consumer_id
+
+        # for each consumer that does not have all the topic partitions it can get
+        # make sure none of the topic partitions it could but did not get cannot be moved to it
+        # (because that would break the balance)
+        for consumer, _ in self.sorted_current_subscriptions:
+            consumer_partition_count = len(self.current_assignment[consumer])
+            # skip if this consumer already has all the topic partitions it can get
+            if consumer_partition_count == len(self.consumer_to_all_potential_partitions[consumer]):
+                continue
+
+            # otherwise make sure it cannot get any more
+            for partition in self.consumer_to_all_potential_partitions[consumer]:
+                if partition not in self.current_assignment[consumer]:
+                    other_consumer = all_assigned_partitions[partition]
+                    other_consumer_partition_count = len(self.current_assignment[other_consumer])
+                    if consumer_partition_count < other_consumer_partition_count:
+                        return False
+        return True
+
+    def _assign_partition(self, partition):
+        for consumer, _ in self.sorted_current_subscriptions:
+            if partition in self.consumer_to_all_potential_partitions[consumer]:
+                self._remove_consumer_from_current_subscriptions_and_maintain_order(consumer)
+                self.current_assignment[consumer].append(partition)
+                self.current_partition_consumer[partition] = consumer
+                self._add_consumer_to_current_subscriptions_and_maintain_order(consumer)
+                break
+
+    def _can_partition_participate_in_reassignment(self, partition):
+        return len(self.partition_to_all_potential_consumers[partition]) >= 2
+
+    def _can_consumer_participate_in_reassignment(self, consumer):
+        current_partitions = self.current_assignment[consumer]
+        current_assignment_size = len(current_partitions)
+        max_assignment_size = len(self.consumer_to_all_potential_partitions[consumer])
+        if current_assignment_size > max_assignment_size:
+            log.error("The consumer {} is assigned more partitions than the maximum possible.".format(consumer))
+        if current_assignment_size < max_assignment_size:
+            # if a consumer is not assigned all its potential partitions it is subject to reassignment
+            return True
+        for partition in current_partitions:
+            # if any of the partitions assigned to a consumer is subject to reassignment the consumer itself
+            # is subject to reassignment
+            if self._can_partition_participate_in_reassignment(partition):
+                return True
+        return False
+
+    def _perform_reassignments(self, reassignable_partitions):
+        reassignment_performed = False
+
+        # repeat reassignment until no partition can be moved to improve the balance
+        while True:
+            modified = False
+            # reassign all reassignable partitions until the full list is processed or a balance is achieved
+            # (starting from the partition with least potential consumers and if needed)
+            for partition in reassignable_partitions:
+                if self._is_balanced():
+                    break
+                # the partition must have at least two potential consumers
+                if len(self.partition_to_all_potential_consumers[partition]) <= 1:
+                    log.error("Expected more than one potential consumer for partition {}".format(partition))
+                # the partition must have a current consumer
+                consumer = self.current_partition_consumer.get(partition)
+                if consumer is None:
+                    log.error("Expected partition {} to be assigned to a consumer".format(partition))
+
+                if (
+                    partition in self.previous_assignment
+                    and len(self.current_assignment[consumer])
+                    > len(self.current_assignment[self.previous_assignment[partition].consumer]) + 1
+                ):
+                    self._reassign_partition_to_consumer(
+                        partition, self.previous_assignment[partition].consumer,
+                    )
+                    reassignment_performed = True
+                    modified = True
+                    continue
+
+                # check if a better-suited consumer exist for the partition; if so, reassign it
+                for other_consumer in self.partition_to_all_potential_consumers[partition]:
+                    if len(self.current_assignment[consumer]) > len(self.current_assignment[other_consumer]) + 1:
+                        self._reassign_partition(partition)
+                        reassignment_performed = True
+                        modified = True
+                        break
+
+            if not modified:
+                break
+        return reassignment_performed
+
+    def _reassign_partition(self, partition):
+        new_consumer = None
+        for another_consumer, _ in self.sorted_current_subscriptions:
+            if partition in self.consumer_to_all_potential_partitions[another_consumer]:
+                new_consumer = another_consumer
+                break
+        assert new_consumer is not None
+        self._reassign_partition_to_consumer(partition, new_consumer)
+
+    def _reassign_partition_to_consumer(self, partition, new_consumer):
+        consumer = self.current_partition_consumer[partition]
+        # find the correct partition movement considering the stickiness requirement
+        partition_to_be_moved = self.partition_movements.get_partition_to_be_moved(partition, consumer, new_consumer)
+        self._move_partition(partition_to_be_moved, new_consumer)
+
+    def _move_partition(self, partition, new_consumer):
+        old_consumer = self.current_partition_consumer[partition]
+        self._remove_consumer_from_current_subscriptions_and_maintain_order(old_consumer)
+        self._remove_consumer_from_current_subscriptions_and_maintain_order(new_consumer)
+
+        self.partition_movements.move_partition(partition, old_consumer, new_consumer)
+
+        self.current_assignment[old_consumer].remove(partition)
+        self.current_assignment[new_consumer].append(partition)
+        self.current_partition_consumer[partition] = new_consumer
+
+        self._add_consumer_to_current_subscriptions_and_maintain_order(new_consumer)
+        self._add_consumer_to_current_subscriptions_and_maintain_order(old_consumer)
+
+    @staticmethod
+    def _get_balance_score(assignment):
+        """Calculates a balance score of a give assignment
+        as the sum of assigned partitions size difference of all consumer pairs.
+        A perfectly balanced assignment (with all consumers getting the same number of partitions)
+        has a balance score of 0. Lower balance score indicates a more balanced assignment.
+
+        Arguments:
+          assignment (dict): {consumer: list of assigned topic partitions}
+
+        Returns:
+          the balance score of the assignment
+        """
+        score = 0
+        consumer_to_assignment = {}
+        for consumer_id, partitions in six.iteritems(assignment):
+            consumer_to_assignment[consumer_id] = len(partitions)
+
+        consumers_to_explore = set(consumer_to_assignment.keys())
+        for consumer_id in consumer_to_assignment.keys():
+            if consumer_id in consumers_to_explore:
+                consumers_to_explore.remove(consumer_id)
+                for other_consumer_id in consumers_to_explore:
+                    score += abs(consumer_to_assignment[consumer_id] - consumer_to_assignment[other_consumer_id])
+        return score
+
+
+class StickyPartitionAssignor(AbstractPartitionAssignor):
+    """
+    https://cwiki.apache.org/confluence/display/KAFKA/KIP-54+-+Sticky+Partition+Assignment+Strategy
+    
+    The sticky assignor serves two purposes. First, it guarantees an assignment that is as balanced as possible, meaning either:
+    - the numbers of topic partitions assigned to consumers differ by at most one; or
+    - each consumer that has 2+ fewer topic partitions than some other consumer cannot get any of those topic partitions transferred to it.
+    
+    Second, it preserved as many existing assignment as possible when a reassignment occurs.
+    This helps in saving some of the overhead processing when topic partitions move from one consumer to another.
+    
+    Starting fresh it would work by distributing the partitions over consumers as evenly as possible.
+    Even though this may sound similar to how round robin assignor works, the second example below shows that it is not.
+    During a reassignment it would perform the reassignment in such a way that in the new assignment
+    - topic partitions are still distributed as evenly as possible, and
+    - topic partitions stay with their previously assigned consumers as much as possible.
+    
+    The first goal above takes precedence over the second one.
+    
+    Example 1.
+    Suppose there are three consumers C0, C1, C2,
+    four topics t0, t1, t2, t3, and each topic has 2 partitions,
+    resulting in partitions t0p0, t0p1, t1p0, t1p1, t2p0, t2p1, t3p0, t3p1.
+    Each consumer is subscribed to all three topics.
+    
+    The assignment with both sticky and round robin assignors will be:
+    - C0: [t0p0, t1p1, t3p0]
+    - C1: [t0p1, t2p0, t3p1]
+    - C2: [t1p0, t2p1]
+    
+    Now, let's assume C1 is removed and a reassignment is about to happen. The round robin assignor would produce:
+    - C0: [t0p0, t1p0, t2p0, t3p0]
+    - C2: [t0p1, t1p1, t2p1, t3p1]
+    
+    while the sticky assignor would result in:
+    - C0 [t0p0, t1p1, t3p0, t2p0]
+    - C2 [t1p0, t2p1, t0p1, t3p1]
+    preserving all the previous assignments (unlike the round robin assignor).
+    
+    
+    Example 2.
+    There are three consumers C0, C1, C2,
+    and three topics t0, t1, t2, with 1, 2, and 3 partitions respectively.
+    Therefore, the partitions are t0p0, t1p0, t1p1, t2p0, t2p1, t2p2.
+    C0 is subscribed to t0;
+    C1 is subscribed to t0, t1;
+    and C2 is subscribed to t0, t1, t2.
+    
+    The round robin assignor would come up with the following assignment:
+    - C0 [t0p0]
+    - C1 [t1p0]
+    - C2 [t1p1, t2p0, t2p1, t2p2]
+    
+    which is not as balanced as the assignment suggested by sticky assignor:
+    - C0 [t0p0]
+    - C1 [t1p0, t1p1]
+    - C2 [t2p0, t2p1, t2p2]
+    
+    Now, if consumer C0 is removed, these two assignors would produce the following assignments.
+    Round Robin (preserves 3 partition assignments):
+    - C1 [t0p0, t1p1]
+    - C2 [t1p0, t2p0, t2p1, t2p2]
+    
+    Sticky (preserves 5 partition assignments):
+    - C1 [t1p0, t1p1, t0p0]
+    - C2 [t2p0, t2p1, t2p2]
+    """
+
+    DEFAULT_GENERATION_ID = -1
+
+    name = "sticky"
+    version = 0
+
+    member_assignment = None
+    generation = DEFAULT_GENERATION_ID
+
+    _latest_partition_movements = None
+
+    @classmethod
+    def assign(cls, cluster, members):
+        """Performs group assignment given cluster metadata and member subscriptions
+
+        Arguments:
+            cluster (ClusterMetadata): cluster metadata
+            members (dict of {member_id: MemberMetadata}): decoded metadata for each member in the group.
+
+        Returns:
+          dict: {member_id: MemberAssignment}
+        """
+        members_metadata = {}
+        for consumer, member_metadata in six.iteritems(members):
+            members_metadata[consumer] = cls.parse_member_metadata(member_metadata)
+
+        executor = StickyAssignmentExecutor(cluster, members_metadata)
+        executor.perform_initial_assignment()
+        executor.balance()
+
+        cls._latest_partition_movements = executor.partition_movements
+
+        assignment = {}
+        for member_id in members:
+            assignment[member_id] = ConsumerProtocolMemberAssignment(
+                cls.version, sorted(executor.get_final_assignment(member_id)), b''
+            )
+        return assignment
+
+    @classmethod
+    def parse_member_metadata(cls, metadata):
+        """
+        Parses member metadata into a python object.
+        This implementation only serializes and deserializes the StickyAssignorMemberMetadataV1 user data,
+        since no StickyAssignor written in Python was deployed ever in the wild with version V0, meaning that
+        there is no need to support backward compatibility with V0.
+
+        Arguments:
+          metadata (MemberMetadata): decoded metadata for a member of the group.
+
+        Returns:
+          parsed metadata (StickyAssignorMemberMetadataV1)
+        """
+        user_data = metadata.user_data
+        if not user_data:
+            return StickyAssignorMemberMetadataV1(
+                partitions=[], generation=cls.DEFAULT_GENERATION_ID, subscription=metadata.subscription
+            )
+
+        try:
+            decoded_user_data = StickyAssignorUserDataV1.decode(user_data)
+        except Exception as e:
+            # ignore the consumer's previous assignment if it cannot be parsed
+            log.error("Could not parse member data", e)     # pylint: disable=logging-too-many-args
+            return StickyAssignorMemberMetadataV1(
+                partitions=[], generation=cls.DEFAULT_GENERATION_ID, subscription=metadata.subscription
+            )
+
+        member_partitions = []
+        for topic, partitions in decoded_user_data.previous_assignment:     # pylint: disable=no-member
+            member_partitions.extend([TopicPartition(topic, partition) for partition in partitions])
+        return StickyAssignorMemberMetadataV1(
+            # pylint: disable=no-member
+            partitions=member_partitions, generation=decoded_user_data.generation, subscription=metadata.subscription
+        )
+
+    @classmethod
+    def metadata(cls, topics):
+        return cls._metadata(topics, cls.member_assignment, cls.generation)
+
+    @classmethod
+    def _metadata(cls, topics, member_assignment_partitions, generation=-1):
+        if member_assignment_partitions is None:
+            log.debug("No member assignment available")
+            user_data = b''
+        else:
+            log.debug("Member assignment is available, generating the metadata: generation {}".format(cls.generation))
+            partitions_by_topic = defaultdict(list)
+            for topic_partition in member_assignment_partitions:
+                partitions_by_topic[topic_partition.topic].append(topic_partition.partition)
+            data = StickyAssignorUserDataV1(list(partitions_by_topic.items()), generation)
+            user_data = data.encode()
+        return ConsumerProtocolMemberMetadata(cls.version, list(topics), user_data)
+
+    @classmethod
+    def on_assignment(cls, assignment):
+        """Callback that runs on each assignment. Updates assignor's state.
+
+        Arguments:
+          assignment: MemberAssignment
+        """
+        log.debug("On assignment: assignment={}".format(assignment))
+        cls.member_assignment = assignment.partitions()
+
+    @classmethod
+    def on_generation_assignment(cls, generation):
+        """Callback that runs on each assignment. Updates assignor's generation id.
+
+        Arguments:
+          generation: generation id
+        """
+        log.debug("On generation assignment: generation={}".format(generation))
+        cls.generation = generation

venv/lib/python3.11/site-packages/kafka/coordinator/consumer.py

@@ -0,0 +1,968 @@
+from __future__ import absolute_import, division
+
+import collections
+import copy
+import functools
+import logging
+import time
+
+from kafka.vendor import six
+
+from kafka.coordinator.base import BaseCoordinator, Generation
+from kafka.coordinator.assignors.range import RangePartitionAssignor
+from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
+from kafka.coordinator.assignors.sticky.sticky_assignor import StickyPartitionAssignor
+from kafka.coordinator.protocol import ConsumerProtocol
+import kafka.errors as Errors
+from kafka.future import Future
+from kafka.metrics import AnonMeasurable
+from kafka.metrics.stats import Avg, Count, Max, Rate
+from kafka.protocol.commit import OffsetCommitRequest, OffsetFetchRequest
+from kafka.structs import OffsetAndMetadata, TopicPartition
+from kafka.util import Timer, WeakMethod
+
+
+log = logging.getLogger(__name__)
+
+
+class ConsumerCoordinator(BaseCoordinator):
+    """This class manages the coordination process with the consumer coordinator."""
+    DEFAULT_CONFIG = {
+        'group_id': 'kafka-python-default-group',
+        'enable_auto_commit': True,
+        'auto_commit_interval_ms': 5000,
+        'default_offset_commit_callback': None,
+        'assignors': (RangePartitionAssignor, RoundRobinPartitionAssignor, StickyPartitionAssignor),
+        'session_timeout_ms': 10000,
+        'heartbeat_interval_ms': 3000,
+        'max_poll_interval_ms': 300000,
+        'retry_backoff_ms': 100,
+        'api_version': (0, 10, 1),
+        'exclude_internal_topics': True,
+        'metrics': None,
+        'metric_group_prefix': 'consumer'
+    }
+
+    def __init__(self, client, subscription, **configs):
+        """Initialize the coordination manager.
+
+        Keyword Arguments:
+            group_id (str): name of the consumer group to join for dynamic
+                partition assignment (if enabled), and to use for fetching and
+                committing offsets. Default: 'kafka-python-default-group'
+            enable_auto_commit (bool): If true the consumer's offset will be
+                periodically committed in the background. Default: True.
+            auto_commit_interval_ms (int): milliseconds between automatic
+                offset commits, if enable_auto_commit is True. Default: 5000.
+            default_offset_commit_callback (callable): called as
+                callback(offsets, response) response will be either an Exception
+                or None. This callback can be used to trigger custom actions when
+                a commit request completes.
+            assignors (list): List of objects to use to distribute partition
+                ownership amongst consumer instances when group management is
+                used. Default: [RangePartitionAssignor, RoundRobinPartitionAssignor]
+            heartbeat_interval_ms (int): The expected time in milliseconds
+                between heartbeats to the consumer coordinator when using
+                Kafka's group management feature. Heartbeats are used to ensure
+                that the consumer's session stays active and to facilitate
+                rebalancing when new consumers join or leave the group. The
+                value must be set lower than session_timeout_ms, but typically
+                should be set no higher than 1/3 of that value. It can be
+                adjusted even lower to control the expected time for normal
+                rebalances. Default: 3000
+            session_timeout_ms (int): The timeout used to detect failures when
+                using Kafka's group management facilities. Default: 30000
+            retry_backoff_ms (int): Milliseconds to backoff when retrying on
+                errors. Default: 100.
+            exclude_internal_topics (bool): Whether records from internal topics
+                (such as offsets) should be exposed to the consumer. If set to
+                True the only way to receive records from an internal topic is
+                subscribing to it. Requires 0.10+. Default: True
+        """
+        super(ConsumerCoordinator, self).__init__(client, **configs)
+
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        for key in self.config:
+            if key in configs:
+                self.config[key] = configs[key]
+
+        self._subscription = subscription
+        self._is_leader = False
+        self._joined_subscription = set()
+        self._metadata_snapshot = self._build_metadata_snapshot(subscription, client.cluster)
+        self._assignment_snapshot = None
+        self._cluster = client.cluster
+        self.auto_commit_interval = self.config['auto_commit_interval_ms'] / 1000
+        self.next_auto_commit_deadline = None
+        self.completed_offset_commits = collections.deque()
+        self._offset_fetch_futures = dict()
+
+        if self.config['default_offset_commit_callback'] is None:
+            self.config['default_offset_commit_callback'] = self._default_offset_commit_callback
+
+        if self.config['group_id'] is not None:
+            if self.config['api_version'] >= (0, 9):
+                if not self.config['assignors']:
+                    raise Errors.KafkaConfigurationError('Coordinator requires assignors')
+            if self.config['api_version'] < (0, 10, 1):
+                if self.config['max_poll_interval_ms'] != self.config['session_timeout_ms']:
+                    raise Errors.KafkaConfigurationError("Broker version %s does not support "
+                                                         "different values for max_poll_interval_ms "
+                                                         "and session_timeout_ms")
+
+        if self.config['enable_auto_commit']:
+            if self.config['api_version'] < (0, 8, 1):
+                log.warning('Broker version (%s) does not support offset'
+                            ' commits; disabling auto-commit.',
+                            self.config['api_version'])
+                self.config['enable_auto_commit'] = False
+            elif self.config['group_id'] is None:
+                log.warning('group_id is None: disabling auto-commit.')
+                self.config['enable_auto_commit'] = False
+            else:
+                self.next_auto_commit_deadline = time.time() + self.auto_commit_interval
+
+        if self.config['metrics']:
+            self._consumer_sensors = ConsumerCoordinatorMetrics(
+                self.config['metrics'], self.config['metric_group_prefix'], self._subscription)
+        else:
+            self._consumer_sensors = None
+
+        self._cluster.request_update()
+        self._cluster.add_listener(WeakMethod(self._handle_metadata_update))
+
+    def __del__(self):
+        if hasattr(self, '_cluster') and self._cluster:
+            try:
+                self._cluster.remove_listener(WeakMethod(self._handle_metadata_update))
+            except TypeError:
+                pass
+        super(ConsumerCoordinator, self).__del__()
+
+    def protocol_type(self):
+        return ConsumerProtocol.PROTOCOL_TYPE
+
+    def group_protocols(self):
+        """Returns list of preferred (protocols, metadata)"""
+        if self._subscription.subscription is None:
+            raise Errors.IllegalStateError('Consumer has not subscribed to topics')
+        # dpkp note: I really dislike this.
+        # why? because we are using this strange method group_protocols,
+        # which is seemingly innocuous, to set internal state (_joined_subscription)
+        # that is later used to check whether metadata has changed since we joined a group
+        # but there is no guarantee that this method, group_protocols, will get called
+        # in the correct sequence or that it will only be called when we want it to be.
+        # So this really should be moved elsewhere, but I don't have the energy to
+        # work that out right now. If you read this at some later date after the mutable
+        # state has bitten you... I'm sorry! It mimics the java client, and that's the
+        # best I've got for now.
+        self._joined_subscription = set(self._subscription.subscription)
+        metadata_list = []
+        for assignor in self.config['assignors']:
+            metadata = assignor.metadata(self._joined_subscription)
+            group_protocol = (assignor.name, metadata)
+            metadata_list.append(group_protocol)
+        return metadata_list
+
+    def _handle_metadata_update(self, cluster):
+        # if we encounter any unauthorized topics, raise an exception
+        if cluster.unauthorized_topics:
+            raise Errors.TopicAuthorizationFailedError(cluster.unauthorized_topics)
+
+        if self._subscription.subscribed_pattern:
+            topics = []
+            for topic in cluster.topics(self.config['exclude_internal_topics']):
+                if self._subscription.subscribed_pattern.match(topic):
+                    topics.append(topic)
+
+            if set(topics) != self._subscription.subscription:
+                self._subscription.change_subscription(topics)
+                self._client.set_topics(self._subscription.group_subscription())
+
+        # check if there are any changes to the metadata which should trigger
+        # a rebalance
+        if self._subscription.partitions_auto_assigned():
+            metadata_snapshot = self._build_metadata_snapshot(self._subscription, cluster)
+            if self._metadata_snapshot != metadata_snapshot:
+                self._metadata_snapshot = metadata_snapshot
+
+                # If we haven't got group coordinator support,
+                # just assign all partitions locally
+                if self._auto_assign_all_partitions():
+                    self._subscription.assign_from_subscribed([
+                        TopicPartition(topic, partition)
+                        for topic in self._subscription.subscription
+                        for partition in self._metadata_snapshot[topic]
+                    ])
+
+    def _auto_assign_all_partitions(self):
+        # For users that use "subscribe" without group support,
+        # we will simply assign all partitions to this consumer
+        if self.config['api_version'] < (0, 9):
+            return True
+        elif self.config['group_id'] is None:
+            return True
+        else:
+            return False
+
+    def _build_metadata_snapshot(self, subscription, cluster):
+        metadata_snapshot = {}
+        for topic in subscription.group_subscription():
+            partitions = cluster.partitions_for_topic(topic)
+            metadata_snapshot[topic] = partitions or set()
+        return metadata_snapshot
+
+    def _lookup_assignor(self, name):
+        for assignor in self.config['assignors']:
+            if assignor.name == name:
+                return assignor
+        return None
+
+    def _on_join_complete(self, generation, member_id, protocol,
+                          member_assignment_bytes):
+        # only the leader is responsible for monitoring for metadata changes
+        # (i.e. partition changes)
+        if not self._is_leader:
+            self._assignment_snapshot = None
+
+        assignor = self._lookup_assignor(protocol)
+        assert assignor, 'Coordinator selected invalid assignment protocol: %s' % (protocol,)
+
+        assignment = ConsumerProtocol.ASSIGNMENT.decode(member_assignment_bytes)
+
+        try:
+            self._subscription.assign_from_subscribed(assignment.partitions())
+        except ValueError as e:
+            log.warning("%s. Probably due to a deleted topic. Requesting Re-join" % e)
+            self.request_rejoin()
+
+        # give the assignor a chance to update internal state
+        # based on the received assignment
+        assignor.on_assignment(assignment)
+        if assignor.name == 'sticky':
+            assignor.on_generation_assignment(generation)
+
+        # reschedule the auto commit starting from now
+        self.next_auto_commit_deadline = time.time() + self.auto_commit_interval
+
+        assigned = set(self._subscription.assigned_partitions())
+        log.info("Setting newly assigned partitions %s for group %s",
+                 assigned, self.group_id)
+
+        # execute the user's callback after rebalance
+        if self._subscription.rebalance_listener:
+            try:
+                self._subscription.rebalance_listener.on_partitions_assigned(assigned)
+            except Exception:
+                log.exception("User provided rebalance listener %s for group %s"
+                              " failed on partition assignment: %s",
+                              self._subscription.rebalance_listener, self.group_id,
+                              assigned)
+
+    def poll(self, timeout_ms=None):
+        """
+        Poll for coordinator events. Only applicable if group_id is set, and
+        broker version supports GroupCoordinators. This ensures that the
+        coordinator is known, and if using automatic partition assignment,
+        ensures that the consumer has joined the group. This also handles
+        periodic offset commits if they are enabled.
+        """
+        if self.group_id is None:
+            return True
+
+        timer = Timer(timeout_ms)
+        try:
+            self._invoke_completed_offset_commit_callbacks()
+            if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
+                log.debug('coordinator.poll: timeout in ensure_coordinator_ready; returning early')
+                return False
+
+            if self.config['api_version'] >= (0, 9) and self._subscription.partitions_auto_assigned():
+                if self.need_rejoin():
+                    # due to a race condition between the initial metadata fetch and the
+                    # initial rebalance, we need to ensure that the metadata is fresh
+                    # before joining initially, and then request the metadata update. If
+                    # metadata update arrives while the rebalance is still pending (for
+                    # example, when the join group is still inflight), then we will lose
+                    # track of the fact that we need to rebalance again to reflect the
+                    # change to the topic subscription. Without ensuring that the
+                    # metadata is fresh, any metadata update that changes the topic
+                    # subscriptions and arrives while a rebalance is in progress will
+                    # essentially be ignored. See KAFKA-3949 for the complete
+                    # description of the problem.
+                    if self._subscription.subscribed_pattern:
+                        metadata_update = self._client.cluster.request_update()
+                        self._client.poll(future=metadata_update, timeout_ms=timer.timeout_ms)
+                        if not metadata_update.is_done:
+                            log.debug('coordinator.poll: timeout updating metadata; returning early')
+                            return False
+
+                    if not self.ensure_active_group(timeout_ms=timer.timeout_ms):
+                        log.debug('coordinator.poll: timeout in ensure_active_group; returning early')
+                        return False
+
+                self.poll_heartbeat()
+
+            self._maybe_auto_commit_offsets_async()
+            return True
+
+        except Errors.KafkaTimeoutError:
+            return False
+
+    def time_to_next_poll(self):
+        """Return seconds (float) remaining until :meth:`.poll` should be called again"""
+        if not self.config['enable_auto_commit']:
+            return self.time_to_next_heartbeat()
+
+        if time.time() > self.next_auto_commit_deadline:
+            return 0
+
+        return min(self.next_auto_commit_deadline - time.time(),
+                   self.time_to_next_heartbeat())
+
+    def _perform_assignment(self, leader_id, assignment_strategy, members):
+        assignor = self._lookup_assignor(assignment_strategy)
+        assert assignor, 'Invalid assignment protocol: %s' % (assignment_strategy,)
+        member_metadata = {}
+        all_subscribed_topics = set()
+        for member_id, metadata_bytes in members:
+            metadata = ConsumerProtocol.METADATA.decode(metadata_bytes)
+            member_metadata[member_id] = metadata
+            all_subscribed_topics.update(metadata.subscription) # pylint: disable-msg=no-member
+
+        # the leader will begin watching for changes to any of the topics
+        # the group is interested in, which ensures that all metadata changes
+        # will eventually be seen
+        # Because assignment typically happens within response callbacks,
+        # we cannot block on metadata updates here (no recursion into poll())
+        self._subscription.group_subscribe(all_subscribed_topics)
+        self._client.set_topics(self._subscription.group_subscription())
+
+        # keep track of the metadata used for assignment so that we can check
+        # after rebalance completion whether anything has changed
+        self._cluster.request_update()
+        self._is_leader = True
+        self._assignment_snapshot = self._metadata_snapshot
+
+        log.debug("Performing assignment for group %s using strategy %s"
+                  " with subscriptions %s", self.group_id, assignor.name,
+                  member_metadata)
+
+        assignments = assignor.assign(self._cluster, member_metadata)
+
+        log.debug("Finished assignment for group %s: %s", self.group_id, assignments)
+
+        group_assignment = {}
+        for member_id, assignment in six.iteritems(assignments):
+            group_assignment[member_id] = assignment
+        return group_assignment
+
+    def _on_join_prepare(self, generation, member_id, timeout_ms=None):
+        # commit offsets prior to rebalance if auto-commit enabled
+        self._maybe_auto_commit_offsets_sync(timeout_ms=timeout_ms)
+
+        # execute the user's callback before rebalance
+        log.info("Revoking previously assigned partitions %s for group %s",
+                 self._subscription.assigned_partitions(), self.group_id)
+        if self._subscription.rebalance_listener:
+            try:
+                revoked = set(self._subscription.assigned_partitions())
+                self._subscription.rebalance_listener.on_partitions_revoked(revoked)
+            except Exception:
+                log.exception("User provided subscription rebalance listener %s"
+                              " for group %s failed on_partitions_revoked",
+                              self._subscription.rebalance_listener, self.group_id)
+
+        self._is_leader = False
+        self._subscription.reset_group_subscription()
+
+    def need_rejoin(self):
+        """Check whether the group should be rejoined
+
+        Returns:
+            bool: True if consumer should rejoin group, False otherwise
+        """
+        if not self._subscription.partitions_auto_assigned():
+            return False
+
+        if self._auto_assign_all_partitions():
+            return False
+
+        # we need to rejoin if we performed the assignment and metadata has changed
+        if (self._assignment_snapshot is not None
+            and self._assignment_snapshot != self._metadata_snapshot):
+            return True
+
+        # we need to join if our subscription has changed since the last join
+        if (self._joined_subscription is not None
+            and self._joined_subscription != self._subscription.subscription):
+            return True
+
+        return super(ConsumerCoordinator, self).need_rejoin()
+
+    def refresh_committed_offsets_if_needed(self, timeout_ms=None):
+        """Fetch committed offsets for assigned partitions."""
+        missing_fetch_positions = set(self._subscription.missing_fetch_positions())
+        try:
+            offsets = self.fetch_committed_offsets(missing_fetch_positions, timeout_ms=timeout_ms)
+        except Errors.KafkaTimeoutError:
+            return False
+        for partition, offset in six.iteritems(offsets):
+            log.debug("Setting offset for partition %s to the committed offset %s", partition, offset.offset)
+            self._subscription.seek(partition, offset.offset)
+        return True
+
+    def fetch_committed_offsets(self, partitions, timeout_ms=None):
+        """Fetch the current committed offsets for specified partitions
+
+        Arguments:
+            partitions (list of TopicPartition): partitions to fetch
+
+        Returns:
+            dict: {TopicPartition: OffsetAndMetadata}
+
+        Raises:
+            KafkaTimeoutError if timeout_ms provided
+        """
+        if not partitions:
+            return {}
+
+        future_key = frozenset(partitions)
+        timer = Timer(timeout_ms)
+        while True:
+            if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
+                timer.maybe_raise()
+
+            # contact coordinator to fetch committed offsets
+            if future_key in self._offset_fetch_futures:
+                future = self._offset_fetch_futures[future_key]
+            else:
+                future = self._send_offset_fetch_request(partitions)
+                self._offset_fetch_futures[future_key] = future
+
+            self._client.poll(future=future, timeout_ms=timer.timeout_ms)
+
+            if future.is_done:
+                del self._offset_fetch_futures[future_key]
+
+                if future.succeeded():
+                    return future.value
+
+                elif not future.retriable():
+                    raise future.exception # pylint: disable-msg=raising-bad-type
+
+            # future failed but is retriable, or is not done yet
+            if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
+                time.sleep(self.config['retry_backoff_ms'] / 1000)
+            else:
+                time.sleep(timer.timeout_ms / 1000)
+            timer.maybe_raise()
+
+    def close(self, autocommit=True, timeout_ms=None):
+        """Close the coordinator, leave the current group,
+        and reset local generation / member_id.
+
+        Keyword Arguments:
+            autocommit (bool): If auto-commit is configured for this consumer,
+                this optional flag causes the consumer to attempt to commit any
+                pending consumed offsets prior to close. Default: True
+        """
+        try:
+            if autocommit:
+                self._maybe_auto_commit_offsets_sync(timeout_ms=timeout_ms)
+        finally:
+            super(ConsumerCoordinator, self).close(timeout_ms=timeout_ms)
+
+    def _invoke_completed_offset_commit_callbacks(self):
+        while self.completed_offset_commits:
+            callback, offsets, res_or_exc = self.completed_offset_commits.popleft()
+            callback(offsets, res_or_exc)
+
+    def commit_offsets_async(self, offsets, callback=None):
+        """Commit specific offsets asynchronously.
+
+        Arguments:
+            offsets (dict {TopicPartition: OffsetAndMetadata}): what to commit
+            callback (callable, optional): called as callback(offsets, response)
+                response will be either an Exception or a OffsetCommitResponse
+                struct. This callback can be used to trigger custom actions when
+                a commit request completes.
+
+        Returns:
+            kafka.future.Future
+        """
+        self._invoke_completed_offset_commit_callbacks()
+        if not self.coordinator_unknown():
+            future = self._do_commit_offsets_async(offsets, callback)
+        else:
+            # we don't know the current coordinator, so try to find it and then
+            # send the commit or fail (we don't want recursive retries which can
+            # cause offset commits to arrive out of order). Note that there may
+            # be multiple offset commits chained to the same coordinator lookup
+            # request. This is fine because the listeners will be invoked in the
+            # same order that they were added. Note also that BaseCoordinator
+            # prevents multiple concurrent coordinator lookup requests.
+            future = self.lookup_coordinator()
+            future.add_callback(lambda r: functools.partial(self._do_commit_offsets_async, offsets, callback)())
+            if callback:
+                future.add_errback(lambda e: self.completed_offset_commits.appendleft((callback, offsets, e)))
+
+        # ensure the commit has a chance to be transmitted (without blocking on
+        # its completion). Note that commits are treated as heartbeats by the
+        # coordinator, so there is no need to explicitly allow heartbeats
+        # through delayed task execution.
+        self._client.poll(timeout_ms=0) # no wakeup if we add that feature
+
+        return future
+
+    def _do_commit_offsets_async(self, offsets, callback=None):
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.UnsupportedVersionError('OffsetCommitRequest requires 0.8.1+ broker')
+        assert all(map(lambda k: isinstance(k, TopicPartition), offsets))
+        assert all(map(lambda v: isinstance(v, OffsetAndMetadata),
+                       offsets.values()))
+        if callback is None:
+            callback = self.config['default_offset_commit_callback']
+        future = self._send_offset_commit_request(offsets)
+        future.add_both(lambda res: self.completed_offset_commits.appendleft((callback, offsets, res)))
+        return future
+
+    def commit_offsets_sync(self, offsets, timeout_ms=None):
+        """Commit specific offsets synchronously.
+
+        This method will retry until the commit completes successfully or an
+        unrecoverable error is encountered.
+
+        Arguments:
+            offsets (dict {TopicPartition: OffsetAndMetadata}): what to commit
+
+        Raises error on failure
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.UnsupportedVersionError('OffsetCommitRequest requires 0.8.1+ broker')
+        assert all(map(lambda k: isinstance(k, TopicPartition), offsets))
+        assert all(map(lambda v: isinstance(v, OffsetAndMetadata),
+                       offsets.values()))
+        self._invoke_completed_offset_commit_callbacks()
+        if not offsets:
+            return
+
+        timer = Timer(timeout_ms)
+        while True:
+            self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms)
+
+            future = self._send_offset_commit_request(offsets)
+            self._client.poll(future=future, timeout_ms=timer.timeout_ms)
+
+            if future.is_done:
+                if future.succeeded():
+                    return future.value
+
+                elif not future.retriable():
+                    raise future.exception # pylint: disable-msg=raising-bad-type
+
+            # future failed but is retriable, or it is still pending
+            if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
+                time.sleep(self.config['retry_backoff_ms'] / 1000)
+            else:
+                time.sleep(timer.timeout_ms / 1000)
+            timer.maybe_raise()
+
+    def _maybe_auto_commit_offsets_sync(self, timeout_ms=None):
+        if self.config['enable_auto_commit']:
+            try:
+                self.commit_offsets_sync(self._subscription.all_consumed_offsets(), timeout_ms=timeout_ms)
+
+            # The three main group membership errors are known and should not
+            # require a stacktrace -- just a warning
+            except (Errors.UnknownMemberIdError,
+                    Errors.IllegalGenerationError,
+                    Errors.RebalanceInProgressError):
+                log.warning("Offset commit failed: group membership out of date"
+                            " This is likely to cause duplicate message"
+                            " delivery.")
+            except Exception:
+                log.exception("Offset commit failed: This is likely to cause"
+                              " duplicate message delivery")
+
+    def _send_offset_commit_request(self, offsets):
+        """Commit offsets for the specified list of topics and partitions.
+
+        This is a non-blocking call which returns a request future that can be
+        polled in the case of a synchronous commit or ignored in the
+        asynchronous case.
+
+        Arguments:
+            offsets (dict of {TopicPartition: OffsetAndMetadata}): what should
+                be committed
+
+        Returns:
+            Future: indicating whether the commit was successful or not
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.UnsupportedVersionError('OffsetCommitRequest requires 0.8.1+ broker')
+        assert all(map(lambda k: isinstance(k, TopicPartition), offsets))
+        assert all(map(lambda v: isinstance(v, OffsetAndMetadata),
+                       offsets.values()))
+        if not offsets:
+            log.debug('No offsets to commit')
+            return Future().success(None)
+
+        node_id = self.coordinator()
+        if node_id is None:
+            return Future().failure(Errors.CoordinatorNotAvailableError)
+
+        # Verify node is ready
+        if not self._client.ready(node_id, metadata_priority=False):
+            log.debug("Node %s not ready -- failing offset commit request",
+                      node_id)
+            return Future().failure(Errors.NodeNotReadyError)
+
+        # create the offset commit request
+        offset_data = collections.defaultdict(dict)
+        for tp, offset in six.iteritems(offsets):
+            offset_data[tp.topic][tp.partition] = offset
+
+        version = self._client.api_version(OffsetCommitRequest, max_version=6)
+        if version > 1 and self._subscription.partitions_auto_assigned():
+            generation = self.generation_if_stable()
+        else:
+            generation = Generation.NO_GENERATION
+
+        # if the generation is None, we are not part of an active group
+        # (and we expect to be). The only thing we can do is fail the commit
+        # and let the user rejoin the group in poll()
+        if generation is None:
+            log.info("Failing OffsetCommit request since the consumer is not part of an active group")
+            if self.rebalance_in_progress():
+                # if the client knows it is already rebalancing, we can use RebalanceInProgressError instead of
+                # CommitFailedError to indicate this is not a fatal error
+                return Future().failure(Errors.RebalanceInProgressError(
+                    "Offset commit cannot be completed since the"
+                    " consumer is undergoing a rebalance for auto partition assignment. You can try completing the rebalance"
+                    " by calling poll() and then retry the operation."))
+            else:
+                return Future().failure(Errors.CommitFailedError(
+                    "Offset commit cannot be completed since the"
+                    " consumer is not part of an active group for auto partition assignment; it is likely that the consumer"
+                    " was kicked out of the group."))
+
+        if version == 0:
+            request = OffsetCommitRequest[version](
+                self.group_id,
+                [(
+                    topic, [(
+                        partition,
+                        offset.offset,
+                        offset.metadata
+                    ) for partition, offset in six.iteritems(partitions)]
+                ) for topic, partitions in six.iteritems(offset_data)]
+            )
+        elif version == 1:
+            request = OffsetCommitRequest[version](
+                self.group_id,
+                # This api version was only used in v0.8.2, prior to join group apis
+                # so this always ends up as NO_GENERATION
+                generation.generation_id,
+                generation.member_id,
+                [(
+                    topic, [(
+                        partition,
+                        offset.offset,
+                        -1, # timestamp, unused
+                        offset.metadata
+                    ) for partition, offset in six.iteritems(partitions)]
+                ) for topic, partitions in six.iteritems(offset_data)]
+            )
+        elif version <= 4:
+            request = OffsetCommitRequest[version](
+                self.group_id,
+                generation.generation_id,
+                generation.member_id,
+                OffsetCommitRequest[version].DEFAULT_RETENTION_TIME,
+                [(
+                    topic, [(
+                        partition,
+                        offset.offset,
+                        offset.metadata
+                    ) for partition, offset in six.iteritems(partitions)]
+                ) for topic, partitions in six.iteritems(offset_data)]
+            )
+        elif version <= 5:
+            request = OffsetCommitRequest[version](
+                self.group_id,
+                generation.generation_id,
+                generation.member_id,
+                [(
+                    topic, [(
+                        partition,
+                        offset.offset,
+                        offset.metadata
+                    ) for partition, offset in six.iteritems(partitions)]
+                ) for topic, partitions in six.iteritems(offset_data)]
+            )
+        else:
+            request = OffsetCommitRequest[version](
+                self.group_id,
+                generation.generation_id,
+                generation.member_id,
+                [(
+                    topic, [(
+                        partition,
+                        offset.offset,
+                        offset.leader_epoch,
+                        offset.metadata
+                    ) for partition, offset in six.iteritems(partitions)]
+                ) for topic, partitions in six.iteritems(offset_data)]
+            )
+
+        log.debug("Sending offset-commit request with %s for group %s to %s",
+                  offsets, self.group_id, node_id)
+
+        future = Future()
+        _f = self._client.send(node_id, request)
+        _f.add_callback(self._handle_offset_commit_response, offsets, future, time.time())
+        _f.add_errback(self._failed_request, node_id, request, future)
+        return future
+
+    def _handle_offset_commit_response(self, offsets, future, send_time, response):
+        log.debug("Received OffsetCommitResponse: %s", response)
+        # TODO look at adding request_latency_ms to response (like java kafka)
+        if self._consumer_sensors:
+            self._consumer_sensors.commit_latency.record((time.time() - send_time) * 1000)
+        unauthorized_topics = set()
+
+        for topic, partitions in response.topics:
+            for partition, error_code in partitions:
+                tp = TopicPartition(topic, partition)
+                offset = offsets[tp]
+
+                error_type = Errors.for_code(error_code)
+                if error_type is Errors.NoError:
+                    log.debug("Group %s committed offset %s for partition %s",
+                              self.group_id, offset, tp)
+                elif error_type is Errors.GroupAuthorizationFailedError:
+                    log.error("Not authorized to commit offsets for group %s",
+                              self.group_id)
+                    future.failure(error_type(self.group_id))
+                    return
+                elif error_type is Errors.TopicAuthorizationFailedError:
+                    unauthorized_topics.add(topic)
+                elif error_type in (Errors.OffsetMetadataTooLargeError,
+                                    Errors.InvalidCommitOffsetSizeError):
+                    # raise the error to the user
+                    log.debug("OffsetCommit for group %s failed on partition %s"
+                              " %s", self.group_id, tp, error_type.__name__)
+                    future.failure(error_type())
+                    return
+                elif error_type is Errors.CoordinatorLoadInProgressError:
+                    # just retry
+                    log.debug("OffsetCommit for group %s failed: %s",
+                              self.group_id, error_type.__name__)
+                    future.failure(error_type(self.group_id))
+                    return
+                elif error_type in (Errors.CoordinatorNotAvailableError,
+                                    Errors.NotCoordinatorError,
+                                    Errors.RequestTimedOutError):
+                    log.debug("OffsetCommit for group %s failed: %s",
+                              self.group_id, error_type.__name__)
+                    self.coordinator_dead(error_type())
+                    future.failure(error_type(self.group_id))
+                    return
+                elif error_type is Errors.RebalanceInProgressError:
+                    # Consumer never tries to commit offset in between join-group and sync-group,
+                    # and hence on broker-side it is not expected to see a commit offset request
+                    # during CompletingRebalance phase; if it ever happens then broker would return
+                    # this error. In this case we should just treat as a fatal CommitFailed exception.
+                    # However, we do not need to reset generations and just request re-join, such that
+                    # if the caller decides to proceed and poll, it would still try to proceed and re-join normally.
+                    self.request_rejoin()
+                    future.failure(Errors.CommitFailedError(error_type()))
+                    return
+                elif error_type in (Errors.UnknownMemberIdError,
+                                    Errors.IllegalGenerationError):
+                    # need reset generation and re-join group
+                    error = error_type(self.group_id)
+                    log.warning("OffsetCommit for group %s failed: %s",
+                                self.group_id, error)
+                    self.reset_generation()
+                    future.failure(Errors.CommitFailedError(error_type()))
+                    return
+                else:
+                    log.error("Group %s failed to commit partition %s at offset"
+                              " %s: %s", self.group_id, tp, offset,
+                              error_type.__name__)
+                    future.failure(error_type())
+                    return
+
+        if unauthorized_topics:
+            log.error("Not authorized to commit to topics %s for group %s",
+                      unauthorized_topics, self.group_id)
+            future.failure(Errors.TopicAuthorizationFailedError(unauthorized_topics))
+        else:
+            future.success(None)
+
+    def _send_offset_fetch_request(self, partitions):
+        """Fetch the committed offsets for a set of partitions.
+
+        This is a non-blocking call. The returned future can be polled to get
+        the actual offsets returned from the broker.
+
+        Arguments:
+            partitions (list of TopicPartition): the partitions to fetch
+
+        Returns:
+            Future: resolves to dict of offsets: {TopicPartition: OffsetAndMetadata}
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.UnsupportedVersionError('OffsetFetchRequest requires 0.8.1+ broker')
+        assert all(map(lambda k: isinstance(k, TopicPartition), partitions))
+        if not partitions:
+            return Future().success({})
+
+        node_id = self.coordinator()
+        if node_id is None:
+            return Future().failure(Errors.CoordinatorNotAvailableError)
+
+        # Verify node is ready
+        if not self._client.ready(node_id, metadata_priority=False):
+            log.debug("Node %s not ready -- failing offset fetch request",
+                      node_id)
+            return Future().failure(Errors.NodeNotReadyError)
+
+        log.debug("Group %s fetching committed offsets for partitions: %s",
+                  self.group_id, partitions)
+        # construct the request
+        topic_partitions = collections.defaultdict(set)
+        for tp in partitions:
+            topic_partitions[tp.topic].add(tp.partition)
+
+        version = self._client.api_version(OffsetFetchRequest, max_version=5)
+        # Starting in version 2, the request can contain a null topics array to indicate that offsets should be fetched
+        # TODO: support
+        request = OffsetFetchRequest[version](
+            self.group_id,
+            list(topic_partitions.items())
+        )
+
+        # send the request with a callback
+        future = Future()
+        _f = self._client.send(node_id, request)
+        _f.add_callback(self._handle_offset_fetch_response, future)
+        _f.add_errback(self._failed_request, node_id, request, future)
+        return future
+
+    def _handle_offset_fetch_response(self, future, response):
+        log.debug("Received OffsetFetchResponse: %s", response)
+        if response.API_VERSION >= 2 and response.error_code != Errors.NoError.errno:
+            error_type = Errors.for_code(response.error_code)
+            log.debug("Offset fetch failed: %s", error_type.__name__)
+            error = error_type()
+            if error_type is Errors.CoordinatorLoadInProgressError:
+                # Retry
+                future.failure(error)
+            elif error_type is Errors.NotCoordinatorError:
+                # re-discover the coordinator and retry
+                self.coordinator_dead(error)
+                future.failure(error)
+            elif error_type is Errors.GroupAuthorizationFailedError:
+                future.failure(error)
+            else:
+                log.error("Unknown error fetching offsets: %s", error)
+                future.failure(error)
+            return
+
+        offsets = {}
+        for topic, partitions in response.topics:
+            for partition_data in partitions:
+                partition, offset = partition_data[:2]
+                if response.API_VERSION >= 5:
+                    leader_epoch, metadata, error_code = partition_data[2:]
+                else:
+                    metadata, error_code = partition_data[2:]
+                    leader_epoch = -1 # noqa: F841
+                tp = TopicPartition(topic, partition)
+                error_type = Errors.for_code(error_code)
+                if error_type is not Errors.NoError:
+                    error = error_type()
+                    log.debug("Group %s failed to fetch offset for partition"
+                              " %s: %s", self.group_id, tp, error)
+                    if error_type is Errors.CoordinatorLoadInProgressError:
+                        # just retry
+                        future.failure(error)
+                    elif error_type is Errors.NotCoordinatorError:
+                        # re-discover the coordinator and retry
+                        self.coordinator_dead(error)
+                        future.failure(error)
+                    elif error_type is Errors.UnknownTopicOrPartitionError:
+                        log.warning("OffsetFetchRequest -- unknown topic %s"
+                                    " (have you committed any offsets yet?)",
+                                    topic)
+                        continue
+                    else:
+                        log.error("Unknown error fetching offsets for %s: %s",
+                                  tp, error)
+                        future.failure(error)
+                    return
+                elif offset >= 0:
+                    # record the position with the offset
+                    # (-1 indicates no committed offset to fetch)
+                    # TODO: save leader_epoch
+                    offsets[tp] = OffsetAndMetadata(offset, metadata, -1)
+                else:
+                    log.debug("Group %s has no committed offset for partition"
+                              " %s", self.group_id, tp)
+        future.success(offsets)
+
+    def _default_offset_commit_callback(self, offsets, res_or_exc):
+        if isinstance(res_or_exc, Exception):
+            log.warning("Auto offset commit failed for group %s: %s",
+                        self.group_id, res_or_exc)
+        else:
+            log.debug("Completed autocommit of offsets %s for group %s",
+                      offsets, self.group_id)
+
+    def _commit_offsets_async_on_complete(self, offsets, res_or_exc):
+        if isinstance(res_or_exc, Exception) and getattr(res_or_exc, 'retriable', False):
+            self.next_auto_commit_deadline = min(time.time() + self.config['retry_backoff_ms'] / 1000, self.next_auto_commit_deadline)
+        self.config['default_offset_commit_callback'](offsets, res_or_exc)
+
+    def _maybe_auto_commit_offsets_async(self):
+        if self.config['enable_auto_commit']:
+            if self.coordinator_unknown():
+                self.next_auto_commit_deadline = time.time() + self.config['retry_backoff_ms'] / 1000
+            elif time.time() > self.next_auto_commit_deadline:
+                self.next_auto_commit_deadline = time.time() + self.auto_commit_interval
+                self._do_auto_commit_offsets_async()
+
+    def maybe_auto_commit_offsets_now(self):
+        if self.config['enable_auto_commit'] and not self.coordinator_unknown():
+            self._do_auto_commit_offsets_async()
+
+    def _do_auto_commit_offsets_async(self):
+        self.commit_offsets_async(self._subscription.all_consumed_offsets(),
+                                  self._commit_offsets_async_on_complete)
+
+
+class ConsumerCoordinatorMetrics(object):
+    def __init__(self, metrics, metric_group_prefix, subscription):
+        self.metrics = metrics
+        self.metric_group_name = '%s-coordinator-metrics' % (metric_group_prefix,)
+
+        self.commit_latency = metrics.sensor('commit-latency')
+        self.commit_latency.add(metrics.metric_name(
+            'commit-latency-avg', self.metric_group_name,
+            'The average time taken for a commit request'), Avg())
+        self.commit_latency.add(metrics.metric_name(
+            'commit-latency-max', self.metric_group_name,
+            'The max time taken for a commit request'), Max())
+        self.commit_latency.add(metrics.metric_name(
+            'commit-rate', self.metric_group_name,
+            'The number of commit calls per second'), Rate(sampled_stat=Count()))
+
+        num_parts = AnonMeasurable(lambda config, now:
+                                   len(subscription.assigned_partitions()))
+        metrics.add_metric(metrics.metric_name(
+            'assigned-partitions', self.metric_group_name,
+            'The number of partitions currently assigned to this consumer'),
+            num_parts)

venv/lib/python3.11/site-packages/kafka/coordinator/heartbeat.py

@@ -0,0 +1,84 @@
+from __future__ import absolute_import, division
+
+import copy
+import logging
+import time
+
+from kafka.errors import KafkaConfigurationError
+
+log = logging.getLogger(__name__)
+
+
+class Heartbeat(object):
+    DEFAULT_CONFIG = {
+        'group_id': None,
+        'heartbeat_interval_ms': 3000,
+        'session_timeout_ms': 10000,
+        'max_poll_interval_ms': 300000,
+        'retry_backoff_ms': 100,
+    }
+
+    def __init__(self, **configs):
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        for key in self.config:
+            if key in configs:
+                self.config[key] = configs[key]
+
+        if self.config['group_id'] is not None:
+            if self.config['heartbeat_interval_ms'] >= self.config['session_timeout_ms']:
+                raise KafkaConfigurationError('Heartbeat interval must be lower than the session timeout (%s v %s)' % (
+                    self.config['heartbeat_interval_ms'], self.config['session_timeout_ms']))
+            if self.config['heartbeat_interval_ms'] > (self.config['session_timeout_ms'] / 3):
+                log.warning('heartbeat_interval_ms is high relative to session_timeout_ms (%s v %s).'
+                            ' Recommend heartbeat interval less than 1/3rd of session timeout',
+                            self.config['heartbeat_interval_ms'], self.config['session_timeout_ms'])
+
+        self.last_send = -1 * float('inf')
+        self.last_receive = -1 * float('inf')
+        self.last_poll = -1 * float('inf')
+        self.last_reset = time.time()
+        self.heartbeat_failed = None
+
+    def poll(self):
+        self.last_poll = time.time()
+
+    def sent_heartbeat(self):
+        self.last_send = time.time()
+        self.heartbeat_failed = False
+
+    def fail_heartbeat(self):
+        self.heartbeat_failed = True
+
+    def received_heartbeat(self):
+        self.last_receive = time.time()
+
+    def time_to_next_heartbeat(self):
+        """Returns seconds (float) remaining before next heartbeat should be sent"""
+        time_since_last_heartbeat = time.time() - max(self.last_send, self.last_reset)
+        if self.heartbeat_failed:
+            delay_to_next_heartbeat = self.config['retry_backoff_ms'] / 1000
+        else:
+            delay_to_next_heartbeat = self.config['heartbeat_interval_ms'] / 1000
+        return max(0, delay_to_next_heartbeat - time_since_last_heartbeat)
+
+    def should_heartbeat(self):
+        return self.time_to_next_heartbeat() == 0
+
+    def session_timeout_expired(self):
+        last_recv = max(self.last_receive, self.last_reset)
+        return (time.time() - last_recv) > (self.config['session_timeout_ms'] / 1000)
+
+    def reset_timeouts(self):
+        self.last_reset = time.time()
+        self.last_poll = time.time()
+        self.heartbeat_failed = False
+
+    def poll_timeout_expired(self):
+        return (time.time() - self.last_poll) > (self.config['max_poll_interval_ms'] / 1000)
+
+    def __str__(self):
+        return ("<Heartbeat group_id={group_id}"
+                " heartbeat_interval_ms={heartbeat_interval_ms}"
+                " session_timeout_ms={session_timeout_ms}"
+                " max_poll_interval_ms={max_poll_interval_ms}"
+                " retry_backoff_ms={retry_backoff_ms}>").format(**self.config)

venv/lib/python3.11/site-packages/kafka/coordinator/protocol.py

@@ -0,0 +1,33 @@
+from __future__ import absolute_import
+
+from kafka.protocol.struct import Struct
+from kafka.protocol.types import Array, Bytes, Int16, Int32, Schema, String
+from kafka.structs import TopicPartition
+
+
+class ConsumerProtocolMemberMetadata(Struct):
+    SCHEMA = Schema(
+        ('version', Int16),
+        ('subscription', Array(String('utf-8'))),
+        ('user_data', Bytes))
+
+
+class ConsumerProtocolMemberAssignment(Struct):
+    SCHEMA = Schema(
+        ('version', Int16),
+        ('assignment', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32)))),
+        ('user_data', Bytes))
+
+    def partitions(self):
+        return [TopicPartition(topic, partition)
+                for topic, partitions in self.assignment # pylint: disable-msg=no-member
+                for partition in partitions]
+
+
+class ConsumerProtocol(object):
+    PROTOCOL_TYPE = 'consumer'
+    ASSIGNMENT_STRATEGIES = ('range', 'roundrobin')
+    METADATA = ConsumerProtocolMemberMetadata
+    ASSIGNMENT = ConsumerProtocolMemberAssignment

venv/lib/python3.11/site-packages/kafka/future.py

@@ -0,0 +1,94 @@
+from __future__ import absolute_import
+
+import functools
+import logging
+import threading
+
+log = logging.getLogger(__name__)
+
+
+class Future(object):
+    error_on_callbacks = False # and errbacks
+
+    def __init__(self):
+        self.is_done = False
+        self.value = None
+        self.exception = None
+        self._callbacks = []
+        self._errbacks = []
+        self._lock = threading.Lock()
+
+    def succeeded(self):
+        return self.is_done and not bool(self.exception)
+
+    def failed(self):
+        return self.is_done and bool(self.exception)
+
+    def retriable(self):
+        try:
+            return self.exception.retriable
+        except AttributeError:
+            return False
+
+    def success(self, value):
+        assert not self.is_done, 'Future is already complete'
+        with self._lock:
+            self.value = value
+            self.is_done = True
+        if self._callbacks:
+            self._call_backs('callback', self._callbacks, self.value)
+        return self
+
+    def failure(self, e):
+        assert not self.is_done, 'Future is already complete'
+        exception = e if type(e) is not type else e()
+        assert isinstance(exception, BaseException), (
+            'future failed without an exception')
+        with self._lock:
+            self.exception = exception
+            self.is_done = True
+        self._call_backs('errback', self._errbacks, self.exception)
+        return self
+
+    def add_callback(self, f, *args, **kwargs):
+        if args or kwargs:
+            f = functools.partial(f, *args, **kwargs)
+        with self._lock:
+            if not self.is_done:
+                self._callbacks.append(f)
+            elif self.succeeded():
+                self._lock.release()
+                self._call_backs('callback', [f], self.value)
+                self._lock.acquire()
+        return self
+
+    def add_errback(self, f, *args, **kwargs):
+        if args or kwargs:
+            f = functools.partial(f, *args, **kwargs)
+        with self._lock:
+            if not self.is_done:
+                self._errbacks.append(f)
+            elif self.failed():
+                self._lock.release()
+                self._call_backs('errback', [f], self.exception)
+                self._lock.acquire()
+        return self
+
+    def add_both(self, f, *args, **kwargs):
+        self.add_callback(f, *args, **kwargs)
+        self.add_errback(f, *args, **kwargs)
+        return self
+
+    def chain(self, future):
+        self.add_callback(future.success)
+        self.add_errback(future.failure)
+        return self
+
+    def _call_backs(self, back_type, backs, value):
+        for f in backs:
+            try:
+                f(value)
+            except Exception as e:
+                log.exception('Error processing %s', back_type)
+                if self.error_on_callbacks:
+                    raise e

venv/lib/python3.11/site-packages/kafka/metrics/__init__.py

@@ -0,0 +1,15 @@
+from __future__ import absolute_import
+
+from kafka.metrics.compound_stat import NamedMeasurable
+from kafka.metrics.dict_reporter import DictReporter
+from kafka.metrics.kafka_metric import KafkaMetric
+from kafka.metrics.measurable import AnonMeasurable
+from kafka.metrics.metric_config import MetricConfig
+from kafka.metrics.metric_name import MetricName
+from kafka.metrics.metrics import Metrics
+from kafka.metrics.quota import Quota
+
+__all__ = [
+    'AnonMeasurable', 'DictReporter', 'KafkaMetric', 'MetricConfig',
+    'MetricName', 'Metrics', 'NamedMeasurable', 'Quota'
+]

venv/lib/python3.11/site-packages/kafka/metrics/compound_stat.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.metrics.stat import AbstractStat
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractCompoundStat(AbstractStat):
+    """
+    A compound stat is a stat where a single measurement and associated
+    data structure feeds many metrics. This is the example for a
+    histogram which has many associated percentiles.
+    """
+    def stats(self):
+        """
+        Return list of NamedMeasurable
+        """
+        raise NotImplementedError
+
+
+class NamedMeasurable(object):
+    __slots__ = ('_name', '_stat')
+
+    def __init__(self, metric_name, measurable_stat):
+        self._name = metric_name
+        self._stat = measurable_stat
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def stat(self):
+        return self._stat

venv/lib/python3.11/site-packages/kafka/metrics/dict_reporter.py

@@ -0,0 +1,83 @@
+from __future__ import absolute_import
+
+import logging
+import threading
+
+from kafka.metrics.metrics_reporter import AbstractMetricsReporter
+
+logger = logging.getLogger(__name__)
+
+
+class DictReporter(AbstractMetricsReporter):
+    """A basic dictionary based metrics reporter.
+
+    Store all metrics in a two level dictionary of category > name > metric.
+    """
+    def __init__(self, prefix=''):
+        self._lock = threading.Lock()
+        self._prefix = prefix if prefix else ''  # never allow None
+        self._store = {}
+
+    def snapshot(self):
+        """
+        Return a nested dictionary snapshot of all metrics and their
+        values at this time. Example:
+        {
+            'category': {
+                'metric1_name': 42.0,
+                'metric2_name': 'foo'
+            }
+        }
+        """
+        return dict((category, dict((name, metric.value())
+                                    for name, metric in list(metrics.items())))
+                    for category, metrics in
+                    list(self._store.items()))
+
+    def init(self, metrics):
+        for metric in metrics:
+            self.metric_change(metric)
+
+    def metric_change(self, metric):
+        with self._lock:
+            category = self.get_category(metric)
+            if category not in self._store:
+                self._store[category] = {}
+            self._store[category][metric.metric_name.name] = metric
+
+    def metric_removal(self, metric):
+        with self._lock:
+            category = self.get_category(metric)
+            metrics = self._store.get(category, {})
+            removed = metrics.pop(metric.metric_name.name, None)
+            if not metrics:
+                self._store.pop(category, None)
+            return removed
+
+    def get_category(self, metric):
+        """
+        Return a string category for the metric.
+
+        The category is made up of this reporter's prefix and the
+        metric's group and tags.
+
+        Examples:
+            prefix = 'foo', group = 'bar', tags = {'a': 1, 'b': 2}
+            returns: 'foo.bar.a=1,b=2'
+
+            prefix = 'foo', group = 'bar', tags = None
+            returns: 'foo.bar'
+
+            prefix = None, group = 'bar', tags = None
+            returns: 'bar'
+        """
+        tags = ','.join('%s=%s' % (k, v) for k, v in
+                        sorted(metric.metric_name.tags.items()))
+        return '.'.join(x for x in
+                        [self._prefix, metric.metric_name.group, tags] if x)
+
+    def configure(self, configs):
+        pass
+
+    def close(self):
+        pass

venv/lib/python3.11/site-packages/kafka/metrics/kafka_metric.py

@@ -0,0 +1,38 @@
+from __future__ import absolute_import
+
+import time
+
+
+class KafkaMetric(object):
+    __slots__ = ('_metric_name', '_measurable', '_config')
+
+    # NOTE java constructor takes a lock instance
+    def __init__(self, metric_name, measurable, config):
+        if not metric_name:
+            raise ValueError('metric_name must be non-empty')
+        if not measurable:
+            raise ValueError('measurable must be non-empty')
+        self._metric_name = metric_name
+        self._measurable = measurable
+        self._config = config
+
+    @property
+    def metric_name(self):
+        return self._metric_name
+
+    @property
+    def measurable(self):
+        return self._measurable
+
+    @property
+    def config(self):
+        return self._config
+
+    @config.setter
+    def config(self, config):
+        self._config = config
+
+    def value(self, time_ms=None):
+        if time_ms is None:
+            time_ms = time.time() * 1000
+        return self._measurable.measure(self._config, time_ms)

venv/lib/python3.11/site-packages/kafka/metrics/measurable.py

@@ -0,0 +1,29 @@
+from __future__ import absolute_import
+
+import abc
+
+
+class AbstractMeasurable(object):
+    """A measurable quantity that can be registered as a metric"""
+    @abc.abstractmethod
+    def measure(self, config, now):
+        """
+        Measure this quantity and return the result
+
+        Arguments:
+            config (MetricConfig): The configuration for this metric
+            now (int): The POSIX time in milliseconds the measurement
+                is being taken
+
+        Returns:
+            The measured value
+        """
+        raise NotImplementedError
+
+
+class AnonMeasurable(AbstractMeasurable):
+    def __init__(self, measure_fn):
+        self._measure_fn = measure_fn
+
+    def measure(self, config, now):
+        return float(self._measure_fn(config, now))

venv/lib/python3.11/site-packages/kafka/metrics/measurable_stat.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.metrics.measurable import AbstractMeasurable
+from kafka.metrics.stat import AbstractStat
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractMeasurableStat(AbstractStat, AbstractMeasurable):
+    """
+    An AbstractMeasurableStat is an AbstractStat that is also
+    an AbstractMeasurable (i.e. can produce a single floating point value).
+    This is the interface used for most of the simple statistics such
+    as Avg, Max, Count, etc.
+    """

venv/lib/python3.11/site-packages/kafka/metrics/metric_config.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import
+
+import sys
+
+
+class MetricConfig(object):
+    """Configuration values for metrics"""
+    __slots__ = ('quota', '_samples', 'event_window', 'time_window_ms', 'tags')
+
+    def __init__(self, quota=None, samples=2, event_window=sys.maxsize,
+                 time_window_ms=30 * 1000, tags=None):
+        """
+        Arguments:
+            quota (Quota, optional): Upper or lower bound of a value.
+            samples (int, optional): Max number of samples kept per metric.
+            event_window (int, optional): Max number of values per sample.
+            time_window_ms (int, optional): Max age of an individual sample.
+            tags (dict of {str: str}, optional): Tags for each metric.
+        """
+        self.quota = quota
+        self._samples = samples
+        self.event_window = event_window
+        self.time_window_ms = time_window_ms
+        # tags should be OrderedDict (not supported in py26)
+        self.tags = tags if tags else {}
+
+    @property
+    def samples(self):
+        return self._samples
+
+    @samples.setter
+    def samples(self, value):
+        if value < 1:
+            raise ValueError('The number of samples must be at least 1.')
+        self._samples = value

venv/lib/python3.11/site-packages/kafka/metrics/metric_name.py

@@ -0,0 +1,107 @@
+from __future__ import absolute_import
+
+import copy
+
+
+class MetricName(object):
+    """
+    This class encapsulates a metric's name, logical group and its
+    related attributes (tags).
+
+    group, tags parameters can be used to create unique metric names.
+    e.g. domainName:type=group,key1=val1,key2=val2
+
+    Usage looks something like this:
+
+        # set up metrics:
+        metric_tags = {'client-id': 'producer-1', 'topic': 'topic'}
+        metric_config = MetricConfig(tags=metric_tags)
+
+        # metrics is the global repository of metrics and sensors
+        metrics = Metrics(metric_config)
+
+        sensor = metrics.sensor('message-sizes')
+        metric_name = metrics.metric_name('message-size-avg',
+                                          'producer-metrics',
+                                          'average message size')
+        sensor.add(metric_name, Avg())
+
+        metric_name = metrics.metric_name('message-size-max',
+        sensor.add(metric_name, Max())
+
+        tags = {'client-id': 'my-client', 'topic': 'my-topic'}
+        metric_name = metrics.metric_name('message-size-min',
+                                          'producer-metrics',
+                                          'message minimum size', tags)
+        sensor.add(metric_name, Min())
+
+        # as messages are sent we record the sizes
+        sensor.record(message_size)
+    """
+    __slots__ = ('_name', '_group', '_description', '_tags', '_hash')
+
+    def __init__(self, name, group, description=None, tags=None):
+        """
+        Arguments:
+            name (str): The name of the metric.
+            group (str): The logical group name of the metrics to which this
+                metric belongs.
+            description (str, optional): A human-readable description to
+                include in the metric.
+            tags (dict, optional): Additional key/val attributes of the metric.
+        """
+        if not (name and group):
+            raise ValueError('name and group must be non-empty.')
+        if tags is not None and not isinstance(tags, dict):
+            raise ValueError('tags must be a dict if present.')
+
+        self._name = name
+        self._group = group
+        self._description = description
+        self._tags = copy.copy(tags)
+        self._hash = 0
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def group(self):
+        return self._group
+
+    @property
+    def description(self):
+        return self._description
+
+    @property
+    def tags(self):
+        return copy.copy(self._tags)
+
+    def __hash__(self):
+        if self._hash != 0:
+            return self._hash
+        prime = 31
+        result = 1
+        result = prime * result + hash(self.group)
+        result = prime * result + hash(self.name)
+        tags_hash = hash(frozenset(self.tags.items())) if self.tags else 0
+        result = prime * result + tags_hash
+        self._hash = result
+        return result
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if other is None:
+            return False
+        return (isinstance(self, type(other)) and
+                self.group == other.group and
+                self.name == other.name and
+                self.tags == other.tags)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __str__(self):
+        return 'MetricName(name=%s, group=%s, description=%s, tags=%s)' % (
+            self.name, self.group, self.description, self.tags)

venv/lib/python3.11/site-packages/kafka/metrics/metrics.py

@@ -0,0 +1,263 @@
+from __future__ import absolute_import
+
+import logging
+import sys
+import time
+import threading
+
+from kafka.metrics import AnonMeasurable, KafkaMetric, MetricConfig, MetricName
+from kafka.metrics.stats import Sensor
+
+logger = logging.getLogger(__name__)
+
+
+class Metrics(object):
+    """
+    A registry of sensors and metrics.
+
+    A metric is a named, numerical measurement. A sensor is a handle to
+    record numerical measurements as they occur. Each Sensor has zero or
+    more associated metrics. For example a Sensor might represent message
+    sizes and we might associate with this sensor a metric for the average,
+    maximum, or other statistics computed off the sequence of message sizes
+    that are recorded by the sensor.
+
+    Usage looks something like this:
+        # set up metrics:
+        metrics = Metrics() # the global repository of metrics and sensors
+        sensor = metrics.sensor('message-sizes')
+        metric_name = MetricName('message-size-avg', 'producer-metrics')
+        sensor.add(metric_name, Avg())
+        metric_name = MetricName('message-size-max', 'producer-metrics')
+        sensor.add(metric_name, Max())
+
+        # as messages are sent we record the sizes
+        sensor.record(message_size);
+    """
+    def __init__(self, default_config=None, reporters=None,
+                 enable_expiration=False):
+        """
+        Create a metrics repository with a default config, given metric
+        reporters and the ability to expire eligible sensors
+
+        Arguments:
+            default_config (MetricConfig, optional): The default config
+            reporters (list of AbstractMetricsReporter, optional):
+                The metrics reporters
+            enable_expiration (bool, optional): true if the metrics instance
+                can garbage collect inactive sensors, false otherwise
+        """
+        self._lock = threading.RLock()
+        self._config = default_config or MetricConfig()
+        self._sensors = {}
+        self._metrics = {}
+        self._children_sensors = {}
+        self._reporters = reporters or []
+        for reporter in self._reporters:
+            reporter.init([])
+        self._closed = False
+
+        if enable_expiration:
+            def expire_loop():
+                while not self._closed:
+                    # delay 30 seconds
+                    time.sleep(30)
+                    self.ExpireSensorTask.run(self)
+            metrics_scheduler = threading.Thread(target=expire_loop)
+            # Creating a daemon thread to not block shutdown
+            metrics_scheduler.daemon = True
+            metrics_scheduler.start()
+
+        self.add_metric(self.metric_name('count', 'kafka-metrics-count',
+                                         'total number of registered metrics'),
+                        AnonMeasurable(lambda config, now: len(self._metrics)))
+
+    @property
+    def config(self):
+        return self._config
+
+    @property
+    def metrics(self):
+        """
+        Get all the metrics currently maintained and indexed by metricName
+        """
+        return self._metrics
+
+    def metric_name(self, name, group, description='', tags=None):
+        """
+        Create a MetricName with the given name, group, description and tags,
+        plus default tags specified in the metric configuration.
+        Tag in tags takes precedence if the same tag key is specified in
+        the default metric configuration.
+
+        Arguments:
+            name (str): The name of the metric
+            group (str): logical group name of the metrics to which this
+                metric belongs
+            description (str, optional): A human-readable description to
+                include in the metric
+            tags (dict, optionals): additional key/value attributes of
+                the metric
+        """
+        combined_tags = dict(self.config.tags)
+        combined_tags.update(tags or {})
+        return MetricName(name, group, description, combined_tags)
+
+    def get_sensor(self, name):
+        """
+        Get the sensor with the given name if it exists
+
+        Arguments:
+            name (str): The name of the sensor
+
+        Returns:
+            Sensor: The sensor or None if no such sensor exists
+        """
+        if not name:
+            raise ValueError('name must be non-empty')
+        return self._sensors.get(name, None)
+
+    def sensor(self, name, config=None,
+               inactive_sensor_expiration_time_seconds=sys.maxsize,
+               parents=None):
+        """
+        Get or create a sensor with the given unique name and zero or
+        more parent sensors. All parent sensors will receive every value
+        recorded with this sensor.
+
+        Arguments:
+            name (str): The name of the sensor
+            config (MetricConfig, optional): A default configuration to use
+                for this sensor for metrics that don't have their own config
+            inactive_sensor_expiration_time_seconds (int, optional):
+                If no value if recorded on the Sensor for this duration of
+                time, it is eligible for removal
+            parents (list of Sensor): The parent sensors
+
+        Returns:
+            Sensor: The sensor that is created
+        """
+        sensor = self.get_sensor(name)
+        if sensor:
+            return sensor
+
+        with self._lock:
+            sensor = self.get_sensor(name)
+            if not sensor:
+                sensor = Sensor(self, name, parents, config or self.config,
+                                inactive_sensor_expiration_time_seconds)
+                self._sensors[name] = sensor
+                if parents:
+                    for parent in parents:
+                        children = self._children_sensors.get(parent)
+                        if not children:
+                            children = []
+                            self._children_sensors[parent] = children
+                        children.append(sensor)
+                logger.debug('Added sensor with name %s', name)
+            return sensor
+
+    def remove_sensor(self, name):
+        """
+        Remove a sensor (if it exists), associated metrics and its children.
+
+        Arguments:
+            name (str): The name of the sensor to be removed
+        """
+        sensor = self._sensors.get(name)
+        if sensor:
+            child_sensors = None
+            with sensor._lock:
+                with self._lock:
+                    val = self._sensors.pop(name, None)
+                    if val and val == sensor:
+                        for metric in sensor.metrics:
+                            self.remove_metric(metric.metric_name)
+                        logger.debug('Removed sensor with name %s', name)
+                        child_sensors = self._children_sensors.pop(sensor, None)
+            if child_sensors:
+                for child_sensor in child_sensors:
+                    self.remove_sensor(child_sensor.name)
+
+    def add_metric(self, metric_name, measurable, config=None):
+        """
+        Add a metric to monitor an object that implements measurable.
+        This metric won't be associated with any sensor.
+        This is a way to expose existing values as metrics.
+
+        Arguments:
+            metricName (MetricName): The name of the metric
+            measurable (AbstractMeasurable): The measurable that will be
+                measured by this metric
+            config (MetricConfig, optional): The configuration to use when
+                measuring this measurable
+        """
+        # NOTE there was a lock here, but i don't think it's needed
+        metric = KafkaMetric(metric_name, measurable, config or self.config)
+        self.register_metric(metric)
+
+    def remove_metric(self, metric_name):
+        """
+        Remove a metric if it exists and return it. Return None otherwise.
+        If a metric is removed, `metric_removal` will be invoked
+        for each reporter.
+
+        Arguments:
+            metric_name (MetricName): The name of the metric
+
+        Returns:
+            KafkaMetric: the removed `KafkaMetric` or None if no such
+                metric exists
+        """
+        with self._lock:
+            metric = self._metrics.pop(metric_name, None)
+            if metric:
+                for reporter in self._reporters:
+                    reporter.metric_removal(metric)
+            return metric
+
+    def add_reporter(self, reporter):
+        """Add a MetricReporter"""
+        with self._lock:
+            reporter.init(list(self.metrics.values()))
+            self._reporters.append(reporter)
+
+    def register_metric(self, metric):
+        with self._lock:
+            if metric.metric_name in self.metrics:
+                raise ValueError('A metric named "%s" already exists, cannot'
+                                 ' register another one.' % (metric.metric_name,))
+            self.metrics[metric.metric_name] = metric
+            for reporter in self._reporters:
+                reporter.metric_change(metric)
+
+    class ExpireSensorTask(object):
+        """
+        This iterates over every Sensor and triggers a remove_sensor
+        if it has expired. Package private for testing
+        """
+        @staticmethod
+        def run(metrics):
+            items = list(metrics._sensors.items())
+            for name, sensor in items:
+                # remove_sensor also locks the sensor object. This is fine
+                # because synchronized is reentrant. There is however a minor
+                # race condition here. Assume we have a parent sensor P and
+                # child sensor C. Calling record on C would cause a record on
+                # P as well. So expiration time for P == expiration time for C.
+                # If the record on P happens via C just after P is removed,
+                # that will cause C to also get removed. Since the expiration
+                # time is typically high it is not expected to be a significant
+                # concern and thus not necessary to optimize
+                with sensor._lock:
+                    if sensor.has_expired():
+                        logger.debug('Removing expired sensor %s', name)
+                        metrics.remove_sensor(name)
+
+    def close(self):
+        """Close this metrics repository."""
+        for reporter in self._reporters:
+            reporter.close()
+
+        self._metrics.clear()
+        self._closed = True

venv/lib/python3.11/site-packages/kafka/metrics/metrics_reporter.py

@@ -0,0 +1,58 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractMetricsReporter(object):
+    """
+    An abstract class to allow things to listen as new metrics
+    are created so they can be reported.
+    """
+    @abc.abstractmethod
+    def init(self, metrics):
+        """
+        This is called when the reporter is first registered
+        to initially register all existing metrics
+
+        Arguments:
+            metrics (list of KafkaMetric): All currently existing metrics
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def metric_change(self, metric):
+        """
+        This is called whenever a metric is updated or added
+
+        Arguments:
+            metric (KafkaMetric)
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def metric_removal(self, metric):
+        """
+        This is called whenever a metric is removed
+
+        Arguments:
+            metric (KafkaMetric)
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def configure(self, configs):
+        """
+        Configure this class with the given key-value pairs
+
+        Arguments:
+            configs (dict of {str, ?})
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def close(self):
+        """Called when the metrics repository is closed."""
+        raise NotImplementedError

venv/lib/python3.11/site-packages/kafka/metrics/quota.py

@@ -0,0 +1,44 @@
+from __future__ import absolute_import
+
+
+class Quota(object):
+    """An upper or lower bound for metrics"""
+    __slots__ = ('_bound', '_upper')
+
+    def __init__(self, bound, is_upper):
+        self._bound = bound
+        self._upper = is_upper
+
+    @staticmethod
+    def upper_bound(upper_bound):
+        return Quota(upper_bound, True)
+
+    @staticmethod
+    def lower_bound(lower_bound):
+        return Quota(lower_bound, False)
+
+    def is_upper_bound(self):
+        return self._upper
+
+    @property
+    def bound(self):
+        return self._bound
+
+    def is_acceptable(self, value):
+        return ((self.is_upper_bound() and value <= self.bound) or
+                (not self.is_upper_bound() and value >= self.bound))
+
+    def __hash__(self):
+        prime = 31
+        result = prime + self.bound
+        return prime * result + self.is_upper_bound()
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        return (isinstance(self, type(other)) and
+                self.bound == other.bound and
+                self.is_upper_bound() == other.is_upper_bound())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)

venv/lib/python3.11/site-packages/kafka/metrics/stat.py

@@ -0,0 +1,24 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractStat(object):
+    """
+    An AbstractStat is a quantity such as average, max, etc that is computed
+    off the stream of updates to a sensor
+    """
+    @abc.abstractmethod
+    def record(self, config, value, time_ms):
+        """
+        Record the given value
+
+        Arguments:
+            config (MetricConfig): The configuration to use for this metric
+            value (float): The value to record
+            timeMs (int): The POSIX time in milliseconds this value occurred
+        """
+        raise NotImplementedError

venv/lib/python3.11/site-packages/kafka/metrics/stats/__init__.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+from kafka.metrics.stats.avg import Avg
+from kafka.metrics.stats.count import Count
+from kafka.metrics.stats.histogram import Histogram
+from kafka.metrics.stats.max_stat import Max
+from kafka.metrics.stats.min_stat import Min
+from kafka.metrics.stats.percentile import Percentile
+from kafka.metrics.stats.percentiles import Percentiles
+from kafka.metrics.stats.rate import Rate
+from kafka.metrics.stats.sensor import Sensor
+from kafka.metrics.stats.total import Total
+
+__all__ = [
+    'Avg', 'Count', 'Histogram', 'Max', 'Min', 'Percentile', 'Percentiles',
+    'Rate', 'Sensor', 'Total'
+]

venv/lib/python3.11/site-packages/kafka/metrics/stats/avg.py

@@ -0,0 +1,26 @@
+from __future__ import absolute_import
+
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class Avg(AbstractSampledStat):
+    """
+    An AbstractSampledStat that maintains a simple average over its samples.
+    """
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self):
+        super(Avg, self).__init__(0.0)
+
+    def update(self, sample, config, value, now):
+        sample.value += value
+
+    def combine(self, samples, config, now):
+        total_sum = 0
+        total_count = 0
+        for sample in samples:
+            total_sum += sample.value
+            total_count += sample.event_count
+        if not total_count:
+            return 0
+        return float(total_sum) / total_count

venv/lib/python3.11/site-packages/kafka/metrics/stats/count.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class Count(AbstractSampledStat):
+    """
+    An AbstractSampledStat that maintains a simple count of what it has seen.
+    """
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self):
+        super(Count, self).__init__(0.0)
+
+    def update(self, sample, config, value, now):
+        sample.value += 1.0
+
+    def combine(self, samples, config, now):
+        return float(sum(sample.value for sample in samples))

venv/lib/python3.11/site-packages/kafka/metrics/stats/histogram.py

@@ -0,0 +1,101 @@
+from __future__ import absolute_import
+
+import math
+
+
+class Histogram(object):
+    __slots__ = ('_hist', '_count', '_bin_scheme')
+
+    def __init__(self, bin_scheme):
+        self._hist = [0.0] * bin_scheme.bins
+        self._count = 0.0
+        self._bin_scheme = bin_scheme
+
+    def record(self, value):
+        self._hist[self._bin_scheme.to_bin(value)] += 1.0
+        self._count += 1.0
+
+    def value(self, quantile):
+        if self._count == 0.0:
+            return float('NaN')
+        _sum = 0.0
+        quant = float(quantile)
+        for i, value in enumerate(self._hist[:-1]):
+            _sum += value
+            if _sum / self._count > quant:
+                return self._bin_scheme.from_bin(i)
+        return float('inf')
+
+    @property
+    def counts(self):
+        return self._hist
+
+    def clear(self):
+        for i in range(self._hist):
+            self._hist[i] = 0.0
+        self._count = 0
+
+    def __str__(self):
+        values = ['%.10f:%.0f' % (self._bin_scheme.from_bin(i), value) for
+                  i, value in enumerate(self._hist[:-1])]
+        values.append('%s:%s' % (float('inf'), self._hist[-1]))
+        return '{%s}' % ','.join(values)
+
+    class ConstantBinScheme(object):
+        __slots__ = ('_min', '_max', '_bins', '_bucket_width')
+
+        def __init__(self, bins, min_val, max_val):
+            if bins < 2:
+                raise ValueError('Must have at least 2 bins.')
+            self._min = float(min_val)
+            self._max = float(max_val)
+            self._bins = int(bins)
+            self._bucket_width = (max_val - min_val) / (bins - 2)
+
+        @property
+        def bins(self):
+            return self._bins
+
+        def from_bin(self, b):
+            if b == 0:
+                return float('-inf')
+            elif b == self._bins - 1:
+                return float('inf')
+            else:
+                return self._min + (b - 1) * self._bucket_width
+
+        def to_bin(self, x):
+            if x < self._min:
+                return 0
+            elif x > self._max:
+                return self._bins - 1
+            else:
+                return int(((x - self._min) / self._bucket_width) + 1)
+
+    class LinearBinScheme(object):
+        __slots__ = ('_bins', '_max', '_scale')
+
+        def __init__(self, num_bins, max_val):
+            self._bins = num_bins
+            self._max = max_val
+            self._scale = max_val / (num_bins * (num_bins - 1) / 2)
+
+        @property
+        def bins(self):
+            return self._bins
+
+        def from_bin(self, b):
+            if b == self._bins - 1:
+                return float('inf')
+            else:
+                unscaled = (b * (b + 1.0)) / 2.0
+                return unscaled * self._scale
+
+        def to_bin(self, x):
+            if x < 0.0:
+                raise ValueError('Values less than 0.0 not accepted.')
+            elif x > self._max:
+                return self._bins - 1
+            else:
+                scaled = x / self._scale
+                return int(-0.5 + math.sqrt(2.0 * scaled + 0.25))

venv/lib/python3.11/site-packages/kafka/metrics/stats/max_stat.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class Max(AbstractSampledStat):
+    """An AbstractSampledStat that gives the max over its samples."""
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self):
+        super(Max, self).__init__(float('-inf'))
+
+    def update(self, sample, config, value, now):
+        sample.value = max(sample.value, value)
+
+    def combine(self, samples, config, now):
+        if not samples:
+            return float('-inf')
+        return float(max(sample.value for sample in samples))

venv/lib/python3.11/site-packages/kafka/metrics/stats/min_stat.py

@@ -0,0 +1,21 @@
+from __future__ import absolute_import
+
+import sys
+
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class Min(AbstractSampledStat):
+    """An AbstractSampledStat that gives the min over its samples."""
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self):
+        super(Min, self).__init__(float(sys.maxsize))
+
+    def update(self, sample, config, value, now):
+        sample.value = min(sample.value, value)
+
+    def combine(self, samples, config, now):
+        if not samples:
+            return float(sys.maxsize)
+        return float(min(sample.value for sample in samples))

venv/lib/python3.11/site-packages/kafka/metrics/stats/percentile.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+
+class Percentile(object):
+    __slots__ = ('_metric_name', '_percentile')
+
+    def __init__(self, metric_name, percentile):
+        self._metric_name = metric_name
+        self._percentile = float(percentile)
+
+    @property
+    def name(self):
+        return self._metric_name
+
+    @property
+    def percentile(self):
+        return self._percentile

venv/lib/python3.11/site-packages/kafka/metrics/stats/percentiles.py

@@ -0,0 +1,77 @@
+from __future__ import absolute_import
+
+from kafka.metrics import AnonMeasurable, NamedMeasurable
+from kafka.metrics.compound_stat import AbstractCompoundStat
+from kafka.metrics.stats import Histogram
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class BucketSizing(object):
+    CONSTANT = 0
+    LINEAR = 1
+
+
+class Percentiles(AbstractSampledStat, AbstractCompoundStat):
+    """A compound stat that reports one or more percentiles"""
+    __slots__ = ('_initial_value', '_samples', '_current',
+                 '_percentiles', '_buckets', '_bin_scheme')
+
+    def __init__(self, size_in_bytes, bucketing, max_val, min_val=0.0,
+                 percentiles=None):
+        super(Percentiles, self).__init__(0.0)
+        self._percentiles = percentiles or []
+        self._buckets = int(size_in_bytes / 4)
+        if bucketing == BucketSizing.CONSTANT:
+            self._bin_scheme = Histogram.ConstantBinScheme(self._buckets,
+                                                           min_val, max_val)
+        elif bucketing == BucketSizing.LINEAR:
+            if min_val != 0.0:
+                raise ValueError('Linear bucket sizing requires min_val'
+                                 ' to be 0.0.')
+            self.bin_scheme = Histogram.LinearBinScheme(self._buckets, max_val)
+        else:
+            raise ValueError('Unknown bucket type: %s' % (bucketing,))
+
+    def stats(self):
+        measurables = []
+
+        def make_measure_fn(pct):
+            return lambda config, now: self.value(config, now,
+                                                  pct / 100.0)
+
+        for percentile in self._percentiles:
+            measure_fn = make_measure_fn(percentile.percentile)
+            stat = NamedMeasurable(percentile.name, AnonMeasurable(measure_fn))
+            measurables.append(stat)
+        return measurables
+
+    def value(self, config, now, quantile):
+        self.purge_obsolete_samples(config, now)
+        count = sum(sample.event_count for sample in self._samples)
+        if count == 0.0:
+            return float('NaN')
+        sum_val = 0.0
+        quant = float(quantile)
+        for b in range(self._buckets):
+            for sample in self._samples:
+                assert type(sample) is self.HistogramSample
+                hist = sample.histogram.counts
+                sum_val += hist[b]
+                if sum_val / count > quant:
+                    return self._bin_scheme.from_bin(b)
+        return float('inf')
+
+    def combine(self, samples, config, now):
+        return self.value(config, now, 0.5)
+
+    def new_sample(self, time_ms):
+        return Percentiles.HistogramSample(self._bin_scheme, time_ms)
+
+    def update(self, sample, config, value, time_ms):
+        assert type(sample) is self.HistogramSample
+        sample.histogram.record(value)
+
+    class HistogramSample(AbstractSampledStat.Sample):
+        def __init__(self, scheme, now):
+            super(Percentiles.HistogramSample, self).__init__(0.0, now)
+            self.histogram = Histogram(scheme)

venv/lib/python3.11/site-packages/kafka/metrics/stats/rate.py

@@ -0,0 +1,120 @@
+from __future__ import absolute_import
+
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class TimeUnit(object):
+    _names = {
+        'nanosecond': 0,
+        'microsecond': 1,
+        'millisecond': 2,
+        'second': 3,
+        'minute': 4,
+        'hour':  5,
+        'day': 6,
+    }
+
+    NANOSECONDS = _names['nanosecond']
+    MICROSECONDS = _names['microsecond']
+    MILLISECONDS = _names['millisecond']
+    SECONDS = _names['second']
+    MINUTES = _names['minute']
+    HOURS = _names['hour']
+    DAYS = _names['day']
+
+    @staticmethod
+    def get_name(time_unit):
+        return TimeUnit._names[time_unit]
+
+
+class Rate(AbstractMeasurableStat):
+    """
+    The rate of the given quantity. By default this is the total observed
+    over a set of samples from a sampled statistic divided by the elapsed
+    time over the sample windows. Alternative AbstractSampledStat
+    implementations can be provided, however, to record the rate of
+    occurrences (e.g. the count of values measured over the time interval)
+    or other such values.
+    """
+    __slots__ = ('_stat', '_unit')
+
+    def __init__(self, time_unit=TimeUnit.SECONDS, sampled_stat=None):
+        self._stat = sampled_stat or SampledTotal()
+        self._unit = time_unit
+
+    def unit_name(self):
+        return TimeUnit.get_name(self._unit)
+
+    def record(self, config, value, time_ms):
+        self._stat.record(config, value, time_ms)
+
+    def measure(self, config, now):
+        value = self._stat.measure(config, now)
+        return float(value) / self.convert(self.window_size(config, now))
+
+    def window_size(self, config, now):
+        # purge old samples before we compute the window size
+        self._stat.purge_obsolete_samples(config, now)
+
+        """
+        Here we check the total amount of time elapsed since the oldest
+        non-obsolete window. This give the total window_size of the batch
+        which is the time used for Rate computation. However, there is
+        an issue if we do not have sufficient data for e.g. if only
+        1 second has elapsed in a 30 second window, the measured rate
+        will be very high. Hence we assume that the elapsed time is
+        always N-1 complete windows plus whatever fraction of the final
+        window is complete.
+
+        Note that we could simply count the amount of time elapsed in
+        the current window and add n-1 windows to get the total time,
+        but this approach does not account for sleeps. AbstractSampledStat
+        only creates samples whenever record is called, if no record is
+        called for a period of time that time is not accounted for in
+        window_size and produces incorrect results.
+        """
+        total_elapsed_time_ms = now - self._stat.oldest(now).last_window_ms
+        # Check how many full windows of data we have currently retained
+        num_full_windows = int(total_elapsed_time_ms / config.time_window_ms)
+        min_full_windows = config.samples - 1
+
+        # If the available windows are less than the minimum required,
+        # add the difference to the totalElapsedTime
+        if num_full_windows < min_full_windows:
+            total_elapsed_time_ms += ((min_full_windows - num_full_windows) *
+                                      config.time_window_ms)
+
+        return total_elapsed_time_ms
+
+    def convert(self, time_ms):
+        if self._unit == TimeUnit.NANOSECONDS:
+            return time_ms * 1000.0 * 1000.0
+        elif self._unit == TimeUnit.MICROSECONDS:
+            return time_ms * 1000.0
+        elif self._unit == TimeUnit.MILLISECONDS:
+            return time_ms
+        elif self._unit == TimeUnit.SECONDS:
+            return time_ms / 1000.0
+        elif self._unit == TimeUnit.MINUTES:
+            return time_ms / (60.0 * 1000.0)
+        elif self._unit == TimeUnit.HOURS:
+            return time_ms / (60.0 * 60.0 * 1000.0)
+        elif self._unit == TimeUnit.DAYS:
+            return time_ms / (24.0 * 60.0 * 60.0 * 1000.0)
+        else:
+            raise ValueError('Unknown unit: %s' % (self._unit,))
+
+
+class SampledTotal(AbstractSampledStat):
+    __slots__ = ('_initial_value', '_samples', '_current')
+    def __init__(self, initial_value=None):
+        if initial_value is not None:
+            raise ValueError('initial_value cannot be set on SampledTotal')
+        super(SampledTotal, self).__init__(0.0)
+
+    def update(self, sample, config, value, time_ms):
+        sample.value += value
+
+    def combine(self, samples, config, now):
+        return float(sum(sample.value for sample in samples))

venv/lib/python3.11/site-packages/kafka/metrics/stats/sampled_stat.py

@@ -0,0 +1,103 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractSampledStat(AbstractMeasurableStat):
+    """
+    An AbstractSampledStat records a single scalar value measured over
+    one or more samples. Each sample is recorded over a configurable
+    window. The window can be defined by number of events or elapsed
+    time (or both, if both are given the window is complete when
+    *either* the event count or elapsed time criterion is met).
+
+    All the samples are combined to produce the measurement. When a
+    window is complete the oldest sample is cleared and recycled to
+    begin recording the next sample.
+
+    Subclasses of this class define different statistics measured
+    using this basic pattern.
+    """
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self, initial_value):
+        self._initial_value = initial_value
+        self._samples = []
+        self._current = 0
+
+    @abc.abstractmethod
+    def update(self, sample, config, value, time_ms):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def combine(self, samples, config, now):
+        raise NotImplementedError
+
+    def record(self, config, value, time_ms):
+        sample = self.current(time_ms)
+        if sample.is_complete(time_ms, config):
+            sample = self._advance(config, time_ms)
+        self.update(sample, config, float(value), time_ms)
+        sample.event_count += 1
+
+    def new_sample(self, time_ms):
+        return self.Sample(self._initial_value, time_ms)
+
+    def measure(self, config, now):
+        self.purge_obsolete_samples(config, now)
+        return float(self.combine(self._samples, config, now))
+
+    def current(self, time_ms):
+        if not self._samples:
+            self._samples.append(self.new_sample(time_ms))
+        return self._samples[self._current]
+
+    def oldest(self, now):
+        if not self._samples:
+            self._samples.append(self.new_sample(now))
+        oldest = self._samples[0]
+        for sample in self._samples[1:]:
+            if sample.last_window_ms < oldest.last_window_ms:
+                oldest = sample
+        return oldest
+
+    def purge_obsolete_samples(self, config, now):
+        """
+        Timeout any windows that have expired in the absence of any events
+        """
+        expire_age = config.samples * config.time_window_ms
+        for sample in self._samples:
+            if now - sample.last_window_ms >= expire_age:
+                sample.reset(now)
+
+    def _advance(self, config, time_ms):
+        self._current = (self._current + 1) % config.samples
+        if self._current >= len(self._samples):
+            sample = self.new_sample(time_ms)
+            self._samples.append(sample)
+            return sample
+        else:
+            sample = self.current(time_ms)
+            sample.reset(time_ms)
+            return sample
+
+    class Sample(object):
+
+        def __init__(self, initial_value, now):
+            self.initial_value = initial_value
+            self.event_count = 0
+            self.last_window_ms = now
+            self.value = initial_value
+
+        def reset(self, now):
+            self.event_count = 0
+            self.last_window_ms = now
+            self.value = self.initial_value
+
+        def is_complete(self, time_ms, config):
+            return (time_ms - self.last_window_ms >= config.time_window_ms or
+                    self.event_count >= config.event_window)

venv/lib/python3.11/site-packages/kafka/metrics/stats/sensor.py

@@ -0,0 +1,138 @@
+from __future__ import absolute_import
+
+import threading
+import time
+
+from kafka.errors import QuotaViolationError
+from kafka.metrics import KafkaMetric
+
+
+class Sensor(object):
+    """
+    A sensor applies a continuous sequence of numerical values
+    to a set of associated metrics. For example a sensor on
+    message size would record a sequence of message sizes using
+    the `record(double)` api and would maintain a set
+    of metrics about request sizes such as the average or max.
+    """
+    __slots__ = ('_lock', '_registry', '_name', '_parents', '_metrics',
+                 '_stats', '_config', '_inactive_sensor_expiration_time_ms',
+                 '_last_record_time')
+
+    def __init__(self, registry, name, parents, config,
+                 inactive_sensor_expiration_time_seconds):
+        if not name:
+            raise ValueError('name must be non-empty')
+        self._lock = threading.RLock()
+        self._registry = registry
+        self._name = name
+        self._parents = parents or []
+        self._metrics = []
+        self._stats = []
+        self._config = config
+        self._inactive_sensor_expiration_time_ms = (
+            inactive_sensor_expiration_time_seconds * 1000)
+        self._last_record_time = time.time() * 1000
+        self._check_forest(set())
+
+    def _check_forest(self, sensors):
+        """Validate that this sensor doesn't end up referencing itself."""
+        if self in sensors:
+            raise ValueError('Circular dependency in sensors: %s is its own'
+                             'parent.' % (self.name,))
+        sensors.add(self)
+        for parent in self._parents:
+            parent._check_forest(sensors)
+
+    @property
+    def name(self):
+        """
+        The name this sensor is registered with.
+        This name will be unique among all registered sensors.
+        """
+        return self._name
+
+    @property
+    def metrics(self):
+        return tuple(self._metrics)
+
+    def record(self, value=1.0, time_ms=None):
+        """
+        Record a value at a known time.
+        Arguments:
+            value (double): The value we are recording
+            time_ms (int): A POSIX timestamp in milliseconds.
+                Default: The time when record() is evaluated (now)
+
+        Raises:
+            QuotaViolationException: if recording this value moves a
+                metric beyond its configured maximum or minimum bound
+        """
+        if time_ms is None:
+            time_ms = time.time() * 1000
+        self._last_record_time = time_ms
+        with self._lock:  # XXX high volume, might be performance issue
+            # increment all the stats
+            for stat in self._stats:
+                stat.record(self._config, value, time_ms)
+            self._check_quotas(time_ms)
+        for parent in self._parents:
+            parent.record(value, time_ms)
+
+    def _check_quotas(self, time_ms):
+        """
+        Check if we have violated our quota for any metric that
+        has a configured quota
+        """
+        for metric in self._metrics:
+            if metric.config and metric.config.quota:
+                value = metric.value(time_ms)
+                if not metric.config.quota.is_acceptable(value):
+                    raise QuotaViolationError("'%s' violated quota. Actual: "
+                                              "%d, Threshold: %d" %
+                                              (metric.metric_name,
+                                               value,
+                                               metric.config.quota.bound))
+
+    def add_compound(self, compound_stat, config=None):
+        """
+        Register a compound statistic with this sensor which
+        yields multiple measurable quantities (like a histogram)
+
+        Arguments:
+            stat (AbstractCompoundStat): The stat to register
+            config (MetricConfig): The configuration for this stat.
+                If None then the stat will use the default configuration
+                for this sensor.
+        """
+        if not compound_stat:
+            raise ValueError('compound stat must be non-empty')
+        self._stats.append(compound_stat)
+        for named_measurable in compound_stat.stats():
+            metric = KafkaMetric(named_measurable.name, named_measurable.stat,
+                                 config or self._config)
+            self._registry.register_metric(metric)
+            self._metrics.append(metric)
+
+    def add(self, metric_name, stat, config=None):
+        """
+        Register a metric with this sensor
+
+        Arguments:
+            metric_name (MetricName): The name of the metric
+            stat (AbstractMeasurableStat): The statistic to keep
+            config (MetricConfig): A special configuration for this metric.
+                If None use the sensor default configuration.
+        """
+        with self._lock:
+            metric = KafkaMetric(metric_name, stat, config or self._config)
+            self._registry.register_metric(metric)
+            self._metrics.append(metric)
+            self._stats.append(stat)
+
+    def has_expired(self):
+        """
+        Return True if the Sensor is eligible for removal due to inactivity.
+        """
+        return ((time.time() * 1000 - self._last_record_time) >
+                self._inactive_sensor_expiration_time_ms)

venv/lib/python3.11/site-packages/kafka/metrics/stats/total.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+
+
+class Total(AbstractMeasurableStat):
+    """An un-windowed cumulative total maintained over all time."""
+    __slots__ = ('_total')
+
+    def __init__(self, value=0.0):
+        self._total = value
+
+    def record(self, config, value, now):
+        self._total += value
+
+    def measure(self, config, now):
+        return float(self._total)

venv/lib/python3.11/site-packages/kafka/partitioner/__init__.py

@@ -0,0 +1,8 @@
+from __future__ import absolute_import
+
+from kafka.partitioner.default import DefaultPartitioner, murmur2
+
+
+__all__ = [
+    'DefaultPartitioner', 'murmur2'
+]

venv/lib/python3.11/site-packages/kafka/partitioner/default.py

@@ -0,0 +1,102 @@
+from __future__ import absolute_import
+
+import random
+
+from kafka.vendor import six
+
+
+class DefaultPartitioner(object):
+    """Default partitioner.
+
+    Hashes key to partition using murmur2 hashing (from java client)
+    If key is None, selects partition randomly from available,
+    or from all partitions if none are currently available
+    """
+    @classmethod
+    def __call__(cls, key, all_partitions, available):
+        """
+        Get the partition corresponding to key
+        :param key: partitioning key
+        :param all_partitions: list of all partitions sorted by partition ID
+        :param available: list of available partitions in no particular order
+        :return: one of the values from all_partitions or available
+        """
+        if key is None:
+            if available:
+                return random.choice(available)
+            return random.choice(all_partitions)
+
+        idx = murmur2(key)
+        idx &= 0x7fffffff
+        idx %= len(all_partitions)
+        return all_partitions[idx]
+
+
+# https://github.com/apache/kafka/blob/0.8.2/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L244
+def murmur2(data):
+    """Pure-python Murmur2 implementation.
+
+    Based on java client, see org.apache.kafka.common.utils.Utils.murmur2
+
+    Args:
+        data (bytes): opaque bytes
+
+    Returns: MurmurHash2 of data
+    """
+    # Python2 bytes is really a str, causing the bitwise operations below to fail
+    # so convert to bytearray.
+    if six.PY2:
+        data = bytearray(bytes(data))
+
+    length = len(data)
+    seed = 0x9747b28c
+    # 'm' and 'r' are mixing constants generated offline.
+    # They're not really 'magic', they just happen to work well.
+    m = 0x5bd1e995
+    r = 24
+
+    # Initialize the hash to a random value
+    h = seed ^ length
+    length4 = length // 4
+
+    for i in range(length4):
+        i4 = i * 4
+        k = ((data[i4 + 0] & 0xff) +
+            ((data[i4 + 1] & 0xff) << 8) +
+            ((data[i4 + 2] & 0xff) << 16) +
+            ((data[i4 + 3] & 0xff) << 24))
+        k &= 0xffffffff
+        k *= m
+        k &= 0xffffffff
+        k ^= (k % 0x100000000) >> r # k ^= k >>> r
+        k &= 0xffffffff
+        k *= m
+        k &= 0xffffffff
+
+        h *= m
+        h &= 0xffffffff
+        h ^= k
+        h &= 0xffffffff
+
+    # Handle the last few bytes of the input array
+    extra_bytes = length % 4
+    if extra_bytes >= 3:
+        h ^= (data[(length & ~3) + 2] & 0xff) << 16
+        h &= 0xffffffff
+    if extra_bytes >= 2:
+        h ^= (data[(length & ~3) + 1] & 0xff) << 8
+        h &= 0xffffffff
+    if extra_bytes >= 1:
+        h ^= (data[length & ~3] & 0xff)
+        h &= 0xffffffff
+        h *= m
+        h &= 0xffffffff
+
+    h ^= (h % 0x100000000) >> 13 # h >>> 13;
+    h &= 0xffffffff
+    h *= m
+    h &= 0xffffffff
+    h ^= (h % 0x100000000) >> 15 # h >>> 15;
+    h &= 0xffffffff
+
+    return h

venv/lib/python3.11/site-packages/kafka/producer/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import
+
+from kafka.producer.kafka import KafkaProducer
+
+__all__ = [
+    'KafkaProducer'
+]

venv/lib/python3.11/site-packages/kafka/producer/future.py

@@ -0,0 +1,71 @@
+from __future__ import absolute_import
+
+import collections
+import threading
+
+from kafka import errors as Errors
+from kafka.future import Future
+
+
+class FutureProduceResult(Future):
+    def __init__(self, topic_partition):
+        super(FutureProduceResult, self).__init__()
+        self.topic_partition = topic_partition
+        self._latch = threading.Event()
+
+    def success(self, value):
+        ret = super(FutureProduceResult, self).success(value)
+        self._latch.set()
+        return ret
+
+    def failure(self, error):
+        ret = super(FutureProduceResult, self).failure(error)
+        self._latch.set()
+        return ret
+
+    def wait(self, timeout=None):
+        # wait() on python2.6 returns None instead of the flag value
+        return self._latch.wait(timeout) or self._latch.is_set()
+
+
+class FutureRecordMetadata(Future):
+    def __init__(self, produce_future, relative_offset, timestamp_ms, checksum, serialized_key_size, serialized_value_size, serialized_header_size):
+        super(FutureRecordMetadata, self).__init__()
+        self._produce_future = produce_future
+        # packing args as a tuple is a minor speed optimization
+        self.args = (relative_offset, timestamp_ms, checksum, serialized_key_size, serialized_value_size, serialized_header_size)
+        produce_future.add_callback(self._produce_success)
+        produce_future.add_errback(self.failure)
+
+    def _produce_success(self, offset_and_timestamp):
+        offset, produce_timestamp_ms = offset_and_timestamp
+
+        # Unpacking from args tuple is minor speed optimization
+        (relative_offset, timestamp_ms, checksum,
+         serialized_key_size, serialized_value_size, serialized_header_size) = self.args
+
+        # None is when Broker does not support the API (<0.10) and
+        # -1 is when the broker is configured for CREATE_TIME timestamps
+        if produce_timestamp_ms is not None and produce_timestamp_ms != -1:
+            timestamp_ms = produce_timestamp_ms
+        if offset != -1 and relative_offset is not None:
+            offset += relative_offset
+        tp = self._produce_future.topic_partition
+        metadata = RecordMetadata(tp[0], tp[1], tp, offset, timestamp_ms,
+                                  checksum, serialized_key_size,
+                                  serialized_value_size, serialized_header_size)
+        self.success(metadata)
+
+    def get(self, timeout=None):
+        if not self.is_done and not self._produce_future.wait(timeout):
+            raise Errors.KafkaTimeoutError(
+                "Timeout after waiting for %s secs." % (timeout,))
+        assert self.is_done
+        if self.failed():
+            raise self.exception # pylint: disable-msg=raising-bad-type
+        return self.value
+
+
+RecordMetadata = collections.namedtuple(
+    'RecordMetadata', ['topic', 'partition', 'topic_partition', 'offset', 'timestamp',
+                       'checksum', 'serialized_key_size', 'serialized_value_size', 'serialized_header_size'])

venv/lib/python3.11/site-packages/kafka/producer/record_accumulator.py

@@ -0,0 +1,674 @@
+from __future__ import absolute_import, division
+
+import collections
+import copy
+import logging
+import threading
+import time
+
+try:
+    # enum in stdlib as of py3.4
+    from enum import IntEnum  # pylint: disable=import-error
+except ImportError:
+    # vendored backport module
+    from kafka.vendor.enum34 import IntEnum
+
+import kafka.errors as Errors
+from kafka.producer.future import FutureRecordMetadata, FutureProduceResult
+from kafka.record.memory_records import MemoryRecordsBuilder
+from kafka.structs import TopicPartition
+
+
+log = logging.getLogger(__name__)
+
+
+class AtomicInteger(object):
+    def __init__(self, val=0):
+        self._lock = threading.Lock()
+        self._val = val
+
+    def increment(self):
+        with self._lock:
+            self._val += 1
+            return self._val
+
+    def decrement(self):
+        with self._lock:
+            self._val -= 1
+            return self._val
+
+    def get(self):
+        return self._val
+
+
+class FinalState(IntEnum):
+    ABORTED = 0
+    FAILED = 1
+    SUCCEEDED = 2
+
+
+class ProducerBatch(object):
+    def __init__(self, tp, records, now=None):
+        now = time.time() if now is None else now
+        self.max_record_size = 0
+        self.created = now
+        self.drained = None
+        self.attempts = 0
+        self.last_attempt = now
+        self.last_append = now
+        self.records = records
+        self.topic_partition = tp
+        self.produce_future = FutureProduceResult(tp)
+        self._retry = False
+        self._final_state = None
+
+    @property
+    def final_state(self):
+        return self._final_state
+
+    @property
+    def record_count(self):
+        return self.records.next_offset()
+
+    @property
+    def producer_id(self):
+        return self.records.producer_id if self.records else None
+
+    @property
+    def producer_epoch(self):
+        return self.records.producer_epoch if self.records else None
+
+    @property
+    def has_sequence(self):
+        return self.records.has_sequence if self.records else False
+
+    def try_append(self, timestamp_ms, key, value, headers, now=None):
+        metadata = self.records.append(timestamp_ms, key, value, headers)
+        if metadata is None:
+            return None
+
+        now = time.time() if now is None else now
+        self.max_record_size = max(self.max_record_size, metadata.size)
+        self.last_append = now
+        future = FutureRecordMetadata(
+            self.produce_future,
+            metadata.offset,
+            metadata.timestamp,
+            metadata.crc,
+            len(key) if key is not None else -1,
+            len(value) if value is not None else -1,
+            sum(len(h_key.encode("utf-8")) + len(h_val) for h_key, h_val in headers) if headers else -1)
+        return future
+
+    def abort(self, exception):
+        """Abort the batch and complete the future and callbacks."""
+        if self._final_state is not None:
+            raise Errors.IllegalStateError("Batch has already been completed in final state: %s" % self._final_state)
+        self._final_state = FinalState.ABORTED
+
+        log.debug("Aborting batch for partition %s: %s", self.topic_partition, exception)
+        self._complete_future(-1, -1, exception)
+
+    def done(self, base_offset=None, timestamp_ms=None, exception=None):
+        """
+        Finalize the state of a batch. Final state, once set, is immutable. This function may be called
+        once or twice on a batch. It may be called twice if
+            1. An inflight batch expires before a response from the broker is received. The batch's final
+            state is set to FAILED. But it could succeed on the broker and second time around batch.done() may
+            try to set SUCCEEDED final state.
+
+            2. If a transaction abortion happens or if the producer is closed forcefully, the final state is
+            ABORTED but again it could succeed if broker responds with a success.
+
+        Attempted transitions from [FAILED | ABORTED] --> SUCCEEDED are logged.
+        Attempted transitions from one failure state to the same or a different failed state are ignored.
+        Attempted transitions from SUCCEEDED to the same or a failed state throw an exception.
+        """
+        final_state = FinalState.SUCCEEDED if exception is None else FinalState.FAILED
+        if self._final_state is None:
+            self._final_state = final_state
+            if final_state is FinalState.SUCCEEDED:
+                log.debug("Successfully produced messages to %s with base offset %s", self.topic_partition, base_offset)
+            else:
+                log.warning("Failed to produce messages to topic-partition %s with base offset %s: %s",
+                            self.topic_partition, base_offset, exception)
+            self._complete_future(base_offset, timestamp_ms, exception)
+            return True
+
+        elif self._final_state is not FinalState.SUCCEEDED:
+            if final_state is FinalState.SUCCEEDED:
+                # Log if a previously unsuccessful batch succeeded later on.
+                log.debug("ProduceResponse returned %s for %s after batch with base offset %s had already been %s.",
+                          final_state, self.topic_partition, base_offset, self._final_state)
+            else:
+                # FAILED --> FAILED and ABORTED --> FAILED transitions are ignored.
+                log.debug("Ignored state transition %s -> %s for %s batch with base offset %s",
+                          self._final_state, final_state, self.topic_partition, base_offset)
+        else:
+            # A SUCCESSFUL batch must not attempt another state change.
+            raise Errors.IllegalStateError("A %s batch must not attempt another state change to %s" % (self._final_state, final_state))
+        return False
+
+    def _complete_future(self, base_offset, timestamp_ms, exception):
+        if self.produce_future.is_done:
+            raise Errors.IllegalStateError('Batch is already closed!')
+        elif exception is None:
+            self.produce_future.success((base_offset, timestamp_ms))
+        else:
+            self.produce_future.failure(exception)
+
+    def has_reached_delivery_timeout(self, delivery_timeout_ms, now=None):
+        now = time.time() if now is None else now
+        return delivery_timeout_ms / 1000 <= now - self.created
+
+    def in_retry(self):
+        return self._retry
+
+    def retry(self, now=None):
+        now = time.time() if now is None else now
+        self._retry = True
+        self.attempts += 1
+        self.last_attempt = now
+        self.last_append = now
+
+    @property
+    def is_done(self):
+        return self.produce_future.is_done
+
+    def __str__(self):
+        return 'ProducerBatch(topic_partition=%s, record_count=%d)' % (
+            self.topic_partition, self.records.next_offset())
+
+
+class RecordAccumulator(object):
+    """
+    This class maintains a dequeue per TopicPartition that accumulates messages
+    into MessageSets to be sent to the server.
+
+    The accumulator attempts to bound memory use, and append calls will block
+    when that memory is exhausted.
+
+    Keyword Arguments:
+        batch_size (int): Requests sent to brokers will contain multiple
+            batches, one for each partition with data available to be sent.
+            A small batch size will make batching less common and may reduce
+            throughput (a batch size of zero will disable batching entirely).
+            Default: 16384
+        compression_attrs (int): The compression type for all data generated by
+            the producer. Valid values are gzip(1), snappy(2), lz4(3), or
+            none(0).
+            Compression is of full batches of data, so the efficacy of batching
+            will also impact the compression ratio (more batching means better
+            compression). Default: None.
+        linger_ms (int): An artificial delay time to add before declaring a
+            record batch (that isn't full) ready for sending. This allows
+            time for more records to arrive. Setting a non-zero linger_ms
+            will trade off some latency for potentially better throughput
+            due to more batching (and hence fewer, larger requests).
+            Default: 0
+        retry_backoff_ms (int): An artificial delay time to retry the
+            produce request upon receiving an error. This avoids exhausting
+            all retries in a short period of time. Default: 100
+    """
+    DEFAULT_CONFIG = {
+        'batch_size': 16384,
+        'compression_attrs': 0,
+        'linger_ms': 0,
+        'request_timeout_ms': 30000,
+        'delivery_timeout_ms': 120000,
+        'retry_backoff_ms': 100,
+        'transaction_manager': None,
+        'message_version': 2,
+    }
+
+    def __init__(self, **configs):
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        for key in self.config:
+            if key in configs:
+                self.config[key] = configs.pop(key)
+
+        self._closed = False
+        self._transaction_manager = self.config['transaction_manager']
+        self._flushes_in_progress = AtomicInteger()
+        self._appends_in_progress = AtomicInteger()
+        self._batches = collections.defaultdict(collections.deque) # TopicPartition: [ProducerBatch]
+        self._tp_locks = {None: threading.Lock()} # TopicPartition: Lock, plus a lock to add entries
+        self._incomplete = IncompleteProducerBatches()
+        # The following variables should only be accessed by the sender thread,
+        # so we don't need to protect them w/ locking.
+        self.muted = set()
+        self._drain_index = 0
+        self._next_batch_expiry_time_ms = float('inf')
+
+        if self.config['delivery_timeout_ms'] < self.config['linger_ms'] + self.config['request_timeout_ms']:
+            raise Errors.KafkaConfigurationError("Must set delivery_timeout_ms higher than linger_ms + request_timeout_ms")
+
+    @property
+    def delivery_timeout_ms(self):
+        return self.config['delivery_timeout_ms']
+
+    @property
+    def next_expiry_time_ms(self):
+        return self._next_batch_expiry_time_ms
+
+    def _tp_lock(self, tp):
+        if tp not in self._tp_locks:
+            with self._tp_locks[None]:
+                if tp not in self._tp_locks:
+                    self._tp_locks[tp] = threading.Lock()
+        return self._tp_locks[tp]
+
+    def append(self, tp, timestamp_ms, key, value, headers, now=None):
+        """Add a record to the accumulator, return the append result.
+
+        The append result will contain the future metadata, and flag for
+        whether the appended batch is full or a new batch is created
+
+        Arguments:
+            tp (TopicPartition): The topic/partition to which this record is
+                being sent
+            timestamp_ms (int): The timestamp of the record (epoch ms)
+            key (bytes): The key for the record
+            value (bytes): The value for the record
+            headers (List[Tuple[str, bytes]]): The header fields for the record
+
+        Returns:
+            tuple: (future, batch_is_full, new_batch_created)
+        """
+        assert isinstance(tp, TopicPartition), 'not TopicPartition'
+        assert not self._closed, 'RecordAccumulator is closed'
+        now = time.time() if now is None else now
+        # We keep track of the number of appending thread to make sure we do
+        # not miss batches in abortIncompleteBatches().
+        self._appends_in_progress.increment()
+        try:
+            with self._tp_lock(tp):
+                # check if we have an in-progress batch
+                dq = self._batches[tp]
+                if dq:
+                    last = dq[-1]
+                    future = last.try_append(timestamp_ms, key, value, headers, now=now)
+                    if future is not None:
+                        batch_is_full = len(dq) > 1 or last.records.is_full()
+                        return future, batch_is_full, False
+
+            with self._tp_lock(tp):
+                # Need to check if producer is closed again after grabbing the
+                # dequeue lock.
+                assert not self._closed, 'RecordAccumulator is closed'
+
+                if dq:
+                    last = dq[-1]
+                    future = last.try_append(timestamp_ms, key, value, headers, now=now)
+                    if future is not None:
+                        # Somebody else found us a batch, return the one we
+                        # waited for! Hopefully this doesn't happen often...
+                        batch_is_full = len(dq) > 1 or last.records.is_full()
+                        return future, batch_is_full, False
+
+                if self._transaction_manager and self.config['message_version'] < 2:
+                    raise Errors.UnsupportedVersionError("Attempting to use idempotence with a broker which"
+                                                         " does not support the required message format (v2)."
+                                                         " The broker must be version 0.11 or later.")
+                records = MemoryRecordsBuilder(
+                    self.config['message_version'],
+                    self.config['compression_attrs'],
+                    self.config['batch_size']
+                )
+
+                batch = ProducerBatch(tp, records, now=now)
+                future = batch.try_append(timestamp_ms, key, value, headers, now=now)
+                if not future:
+                    raise Exception()
+
+                dq.append(batch)
+                self._incomplete.add(batch)
+                batch_is_full = len(dq) > 1 or batch.records.is_full()
+                return future, batch_is_full, True
+        finally:
+            self._appends_in_progress.decrement()
+
+    def reset_next_batch_expiry_time(self):
+        self._next_batch_expiry_time_ms = float('inf')
+
+    def maybe_update_next_batch_expiry_time(self, batch):
+        self._next_batch_expiry_time_ms = min(self._next_batch_expiry_time_ms, batch.created * 1000 + self.delivery_timeout_ms)
+
+    def expired_batches(self, now=None):
+        """Get a list of batches which have been sitting in the accumulator too long and need to be expired."""
+        expired_batches = []
+        for tp in list(self._batches.keys()):
+            with self._tp_lock(tp):
+                # iterate over the batches and expire them if they have stayed
+                # in accumulator for more than request_timeout_ms
+                dq = self._batches[tp]
+                while dq:
+                    batch = dq[0]
+                    if batch.has_reached_delivery_timeout(self.delivery_timeout_ms, now=now):
+                        dq.popleft()
+                        batch.records.close()
+                        expired_batches.append(batch)
+                    else:
+                        # Stop at the first batch that has not expired.
+                        self.maybe_update_next_batch_expiry_time(batch)
+                        break
+        return expired_batches
+
+    def reenqueue(self, batch, now=None):
+        """
+        Re-enqueue the given record batch in the accumulator. In Sender._complete_batch method, we check
+        whether the batch has reached delivery_timeout_ms or not. Hence we do not do the delivery timeout check here.
+        """
+        batch.retry(now=now)
+        with self._tp_lock(batch.topic_partition):
+            dq = self._batches[batch.topic_partition]
+            dq.appendleft(batch)
+
+    def ready(self, cluster, now=None):
+        """
+        Get a list of nodes whose partitions are ready to be sent, and the
+        earliest time at which any non-sendable partition will be ready;
+        Also return the flag for whether there are any unknown leaders for the
+        accumulated partition batches.
+
+        A destination node is ready to send if:
+
+         * There is at least one partition that is not backing off its send
+         * and those partitions are not muted (to prevent reordering if
+           max_in_flight_requests_per_connection is set to 1)
+         * and any of the following are true:
+
+           * The record set is full
+           * The record set has sat in the accumulator for at least linger_ms
+             milliseconds
+           * The accumulator is out of memory and threads are blocking waiting
+             for data (in this case all partitions are immediately considered
+             ready).
+           * The accumulator has been closed
+
+        Arguments:
+            cluster (ClusterMetadata):
+
+        Returns:
+            tuple:
+                ready_nodes (set): node_ids that have ready batches
+                next_ready_check (float): secs until next ready after backoff
+                unknown_leaders_exist (bool): True if metadata refresh needed
+        """
+        ready_nodes = set()
+        next_ready_check = 9999999.99
+        unknown_leaders_exist = False
+        now = time.time() if now is None else now
+
+        # several threads are accessing self._batches -- to simplify
+        # concurrent access, we iterate over a snapshot of partitions
+        # and lock each partition separately as needed
+        partitions = list(self._batches.keys())
+        for tp in partitions:
+            leader = cluster.leader_for_partition(tp)
+            if leader is None or leader == -1:
+                unknown_leaders_exist = True
+                continue
+            elif leader in ready_nodes:
+                continue
+            elif tp in self.muted:
+                continue
+
+            with self._tp_lock(tp):
+                dq = self._batches[tp]
+                if not dq:
+                    continue
+                batch = dq[0]
+                retry_backoff = self.config['retry_backoff_ms'] / 1000
+                linger = self.config['linger_ms'] / 1000
+                backing_off = bool(batch.attempts > 0
+                                   and (batch.last_attempt + retry_backoff) > now)
+                waited_time = now - batch.last_attempt
+                time_to_wait = retry_backoff if backing_off else linger
+                time_left = max(time_to_wait - waited_time, 0)
+                full = bool(len(dq) > 1 or batch.records.is_full())
+                expired = bool(waited_time >= time_to_wait)
+
+                sendable = (full or expired or self._closed or
+                            self.flush_in_progress())
+
+                if sendable and not backing_off:
+                    ready_nodes.add(leader)
+                else:
+                    # Note that this results in a conservative estimate since
+                    # an un-sendable partition may have a leader that will
+                    # later be found to have sendable data. However, this is
+                    # good enough since we'll just wake up and then sleep again
+                    # for the remaining time.
+                    next_ready_check = min(time_left, next_ready_check)
+
+        return ready_nodes, next_ready_check, unknown_leaders_exist
+
+    def has_undrained(self):
+        """Check whether there are any batches which haven't been drained"""
+        for tp in list(self._batches.keys()):
+            with self._tp_lock(tp):
+                dq = self._batches[tp]
+                if len(dq):
+                    return True
+        return False
+
+    def _should_stop_drain_batches_for_partition(self, first, tp):
+        if self._transaction_manager:
+            if not self._transaction_manager.is_send_to_partition_allowed(tp):
+                return True
+            if not self._transaction_manager.producer_id_and_epoch.is_valid:
+                # we cannot send the batch until we have refreshed the PID
+                log.debug("Waiting to send ready batches because transaction producer id is not valid")
+                return True
+        return False
+
+    def drain_batches_for_one_node(self, cluster, node_id, max_size, now=None):
+        now = time.time() if now is None else now
+        size = 0
+        ready = []
+        partitions = list(cluster.partitions_for_broker(node_id))
+        if not partitions:
+            return ready
+        # to make starvation less likely this loop doesn't start at 0
+        self._drain_index %= len(partitions)
+        start = None
+        while start != self._drain_index:
+            tp = partitions[self._drain_index]
+            if start is None:
+                start = self._drain_index
+            self._drain_index += 1
+            self._drain_index %= len(partitions)
+
+            # Only proceed if the partition has no in-flight batches.
+            if tp in self.muted:
+                continue
+
+            if tp not in self._batches:
+                continue
+
+            with self._tp_lock(tp):
+                dq = self._batches[tp]
+                if len(dq) == 0:
+                    continue
+                first = dq[0]
+                backoff = bool(first.attempts > 0 and
+                               first.last_attempt + self.config['retry_backoff_ms'] / 1000 > now)
+                # Only drain the batch if it is not during backoff
+                if backoff:
+                    continue
+
+                if (size + first.records.size_in_bytes() > max_size
+                    and len(ready) > 0):
+                    # there is a rare case that a single batch
+                    # size is larger than the request size due
+                    # to compression; in this case we will
+                    # still eventually send this batch in a
+                    # single request
+                    break
+                else:
+                    if self._should_stop_drain_batches_for_partition(first, tp):
+                        break
+
+                    batch = dq.popleft()
+                    if self._transaction_manager and not batch.in_retry():
+                        # If the batch is in retry, then we should not change the pid and
+                        # sequence number, since this may introduce duplicates. In particular,
+                        # the previous attempt may actually have been accepted, and if we change
+                        # the pid and sequence here, this attempt will also be accepted, causing
+                        # a duplicate.
+                        sequence_number = self._transaction_manager.sequence_number(batch.topic_partition)
+                        log.debug("Dest: %s: %s producer_id=%s epoch=%s sequence=%s",
+                                  node_id, batch.topic_partition,
+                                  self._transaction_manager.producer_id_and_epoch.producer_id,
+                                  self._transaction_manager.producer_id_and_epoch.epoch,
+                                  sequence_number)
+                        batch.records.set_producer_state(
+                            self._transaction_manager.producer_id_and_epoch.producer_id,
+                            self._transaction_manager.producer_id_and_epoch.epoch,
+                            sequence_number,
+                            self._transaction_manager.is_transactional()
+                        )
+                    batch.records.close()
+                    size += batch.records.size_in_bytes()
+                    ready.append(batch)
+                    batch.drained = now
+        return ready
+
+    def drain(self, cluster, nodes, max_size, now=None):
+        """
+        Drain all the data for the given nodes and collate them into a list of
+        batches that will fit within the specified size on a per-node basis.
+        This method attempts to avoid choosing the same topic-node repeatedly.
+
+        Arguments:
+            cluster (ClusterMetadata): The current cluster metadata
+            nodes (list): list of node_ids to drain
+            max_size (int): maximum number of bytes to drain
+
+        Returns:
+            dict: {node_id: list of ProducerBatch} with total size less than the
+                requested max_size.
+        """
+        if not nodes:
+            return {}
+
+        now = time.time() if now is None else now
+        batches = {}
+        for node_id in nodes:
+            batches[node_id] = self.drain_batches_for_one_node(cluster, node_id, max_size, now=now)
+        return batches
+
+    def deallocate(self, batch):
+        """Deallocate the record batch."""
+        self._incomplete.remove(batch)
+
+    def flush_in_progress(self):
+        """Are there any threads currently waiting on a flush?"""
+        return self._flushes_in_progress.get() > 0
+
+    def begin_flush(self):
+        """
+        Initiate the flushing of data from the accumulator...this makes all
+        requests immediately ready
+        """
+        self._flushes_in_progress.increment()
+
+    def await_flush_completion(self, timeout=None):
+        """
+        Mark all partitions as ready to send and block until the send is complete
+        """
+        try:
+            for batch in self._incomplete.all():
+                log.debug('Waiting on produce to %s',
+                          batch.produce_future.topic_partition)
+                if not batch.produce_future.wait(timeout=timeout):
+                    raise Errors.KafkaTimeoutError('Timeout waiting for future')
+                if not batch.produce_future.is_done:
+                    raise Errors.UnknownError('Future not done')
+
+                if batch.produce_future.failed():
+                    log.warning(batch.produce_future.exception)
+        finally:
+            self._flushes_in_progress.decrement()
+
+    @property
+    def has_incomplete(self):
+        return bool(self._incomplete)
+
+    def abort_incomplete_batches(self):
+        """
+        This function is only called when sender is closed forcefully. It will fail all the
+        incomplete batches and return.
+        """
+        # We need to keep aborting the incomplete batch until no thread is trying to append to
+        # 1. Avoid losing batches.
+        # 2. Free up memory in case appending threads are blocked on buffer full.
+        # This is a tight loop but should be able to get through very quickly.
+        error = Errors.IllegalStateError("Producer is closed forcefully.")
+        while True:
+            self._abort_batches(error)
+            if not self._appends_in_progress.get():
+                break
+        # After this point, no thread will append any messages because they will see the close
+        # flag set. We need to do the last abort after no thread was appending in case the there was a new
+        # batch appended by the last appending thread.
+        self._abort_batches(error)
+        self._batches.clear()
+
+    def _abort_batches(self, error):
+        """Go through incomplete batches and abort them."""
+        for batch in self._incomplete.all():
+            tp = batch.topic_partition
+            # Close the batch before aborting
+            with self._tp_lock(tp):
+                batch.records.close()
+                self._batches[tp].remove(batch)
+            batch.abort(error)
+            self.deallocate(batch)
+
+    def abort_undrained_batches(self, error):
+        for batch in self._incomplete.all():
+            tp = batch.topic_partition
+            with self._tp_lock(tp):
+                aborted = False
+                if not batch.is_done:
+                    aborted = True
+                    batch.records.close()
+                    self._batches[tp].remove(batch)
+            if aborted:
+                batch.abort(error)
+                self.deallocate(batch)
+
+    def close(self):
+        """Close this accumulator and force all the record buffers to be drained."""
+        self._closed = True
+
+
+class IncompleteProducerBatches(object):
+    """A threadsafe helper class to hold ProducerBatches that haven't been ack'd yet"""
+
+    def __init__(self):
+        self._incomplete = set()
+        self._lock = threading.Lock()
+
+    def add(self, batch):
+        with self._lock:
+            self._incomplete.add(batch)
+
+    def remove(self, batch):
+        with self._lock:
+            try:
+                self._incomplete.remove(batch)
+            except KeyError:
+                pass
+
+    def all(self):
+        with self._lock:
+            return list(self._incomplete)
+
+    def __bool__(self):
+        return bool(self._incomplete)
+
+
+    __nonzero__ = __bool__

venv/lib/python3.11/site-packages/kafka/producer/sender.py

@@ -0,0 +1,763 @@
+from __future__ import absolute_import, division
+
+import collections
+import copy
+import heapq
+import logging
+import threading
+import time
+
+from kafka.vendor import six
+
+from kafka import errors as Errors
+from kafka.metrics.measurable import AnonMeasurable
+from kafka.metrics.stats import Avg, Max, Rate
+from kafka.producer.transaction_manager import ProducerIdAndEpoch
+from kafka.protocol.init_producer_id import InitProducerIdRequest
+from kafka.protocol.produce import ProduceRequest
+from kafka.structs import TopicPartition
+from kafka.version import __version__
+
+log = logging.getLogger(__name__)
+
+
+class Sender(threading.Thread):
+    """
+    The background thread that handles the sending of produce requests to the
+    Kafka cluster. This thread makes metadata requests to renew its view of the
+    cluster and then sends produce requests to the appropriate nodes.
+    """
+    DEFAULT_CONFIG = {
+        'max_request_size': 1048576,
+        'acks': 1,
+        'retries': float('inf'),
+        'request_timeout_ms': 30000,
+        'retry_backoff_ms': 100,
+        'metrics': None,
+        'guarantee_message_order': False,
+        'transaction_manager': None,
+        'transactional_id': None,
+        'transaction_timeout_ms': 60000,
+        'client_id': 'kafka-python-' + __version__,
+    }
+
+    def __init__(self, client, metadata, accumulator, **configs):
+        super(Sender, self).__init__()
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        for key in self.config:
+            if key in configs:
+                self.config[key] = configs.pop(key)
+
+        self.name = self.config['client_id'] + '-network-thread'
+        self._client = client
+        self._accumulator = accumulator
+        self._metadata = client.cluster
+        self._running = True
+        self._force_close = False
+        self._topics_to_add = set()
+        if self.config['metrics']:
+            self._sensors = SenderMetrics(self.config['metrics'], self._client, self._metadata)
+        else:
+            self._sensors = None
+        self._transaction_manager = self.config['transaction_manager']
+        # A per-partition queue of batches ordered by creation time for tracking the in-flight batches
+        self._in_flight_batches = collections.defaultdict(list)
+
+    def _maybe_remove_from_inflight_batches(self, batch):
+        try:
+            queue = self._in_flight_batches[batch.topic_partition]
+        except KeyError:
+            return
+        try:
+            idx = queue.index((batch.created, batch))
+        except ValueError:
+            return
+        # https://stackoverflow.com/questions/10162679/python-delete-element-from-heap
+        queue[idx] = queue[-1]
+        queue.pop()
+        heapq.heapify(queue)
+
+    def _get_expired_inflight_batches(self, now=None):
+        """Get the in-flight batches that has reached delivery timeout."""
+        expired_batches = []
+        to_remove = []
+        for tp, queue in six.iteritems(self._in_flight_batches):
+            while queue:
+                _created_at, batch = queue[0]
+                if batch.has_reached_delivery_timeout(self._accumulator.delivery_timeout_ms):
+                    heapq.heappop(queue)
+                    if batch.final_state is None:
+                        expired_batches.append(batch)
+                    else:
+                        raise Errors.IllegalStateError("%s batch created at %s gets unexpected final state %s" % (batch.topic_partition, batch.created, batch.final_state))
+                else:
+                    self._accumulator.maybe_update_next_batch_expiry_time(batch)
+                    break
+            else:
+                # Avoid mutating in_flight_batches during iteration
+                to_remove.append(tp)
+        for tp in to_remove:
+            del self._in_flight_batches[tp]
+        return expired_batches
+
+    def run(self):
+        """The main run loop for the sender thread."""
+        log.debug("%s: Starting Kafka producer I/O thread.", str(self))
+
+        # main loop, runs until close is called
+        while self._running:
+            try:
+                self.run_once()
+            except Exception:
+                log.exception("%s: Uncaught error in kafka producer I/O thread", str(self))
+
+        log.debug("%s: Beginning shutdown of Kafka producer I/O thread, sending"
+                  " remaining records.", str(self))
+
+        # okay we stopped accepting requests but there may still be
+        # requests in the accumulator or waiting for acknowledgment,
+        # wait until these are completed.
+        while (not self._force_close
+               and (self._accumulator.has_undrained()
+                    or self._client.in_flight_request_count() > 0)):
+            try:
+                self.run_once()
+            except Exception:
+                log.exception("%s: Uncaught error in kafka producer I/O thread", str(self))
+
+        if self._force_close:
+            # We need to fail all the incomplete batches and wake up the
+            # threads waiting on the futures.
+            self._accumulator.abort_incomplete_batches()
+
+        try:
+            self._client.close()
+        except Exception:
+            log.exception("%s: Failed to close network client", str(self))
+
+        log.debug("%s: Shutdown of Kafka producer I/O thread has completed.", str(self))
+
+    def run_once(self):
+        """Run a single iteration of sending."""
+        while self._topics_to_add:
+            self._client.add_topic(self._topics_to_add.pop())
+
+        if self._transaction_manager:
+            try:
+                if not self._transaction_manager.is_transactional():
+                    # this is an idempotent producer, so make sure we have a producer id
+                    self._maybe_wait_for_producer_id()
+                elif self._transaction_manager.has_in_flight_transactional_request() or self._maybe_send_transactional_request():
+                    # as long as there are outstanding transactional requests, we simply wait for them to return
+                    self._client.poll(timeout_ms=self.config['retry_backoff_ms'])
+                    return
+
+                # do not continue sending if the transaction manager is in a failed state or if there
+                # is no producer id (for the idempotent case).
+                if self._transaction_manager.has_fatal_error() or not self._transaction_manager.has_producer_id():
+                    last_error = self._transaction_manager.last_error
+                    if last_error is not None:
+                        self._maybe_abort_batches(last_error)
+                    self._client.poll(timeout_ms=self.config['retry_backoff_ms'])
+                    return
+                elif self._transaction_manager.has_abortable_error():
+                    self._accumulator.abort_undrained_batches(self._transaction_manager.last_error)
+
+            except Errors.SaslAuthenticationFailedError as e:
+                # This is already logged as error, but propagated here to perform any clean ups.
+                log.debug("%s: Authentication exception while processing transactional request: %s", str(self), e)
+                self._transaction_manager.authentication_failed(e)
+
+        poll_timeout_ms = self._send_producer_data()
+        self._client.poll(timeout_ms=poll_timeout_ms)
+
+    def _send_producer_data(self, now=None):
+        now = time.time() if now is None else now
+        # get the list of partitions with data ready to send
+        result = self._accumulator.ready(self._metadata, now=now)
+        ready_nodes, next_ready_check_delay, unknown_leaders_exist = result
+
+        # if there are any partitions whose leaders are not known yet, force
+        # metadata update
+        if unknown_leaders_exist:
+            log.debug('%s: Unknown leaders exist, requesting metadata update', str(self))
+            self._metadata.request_update()
+
+        # remove any nodes we aren't ready to send to
+        not_ready_timeout_ms = float('inf')
+        for node in list(ready_nodes):
+            if not self._client.is_ready(node):
+                node_delay_ms = self._client.connection_delay(node)
+                log.debug('%s: Node %s not ready; delaying produce of accumulated batch (%f ms)', str(self), node, node_delay_ms)
+                self._client.maybe_connect(node, wakeup=False)
+                ready_nodes.remove(node)
+                not_ready_timeout_ms = min(not_ready_timeout_ms, node_delay_ms)
+
+        # create produce requests
+        batches_by_node = self._accumulator.drain(
+            self._metadata, ready_nodes, self.config['max_request_size'], now=now)
+
+        for batch_list in six.itervalues(batches_by_node):
+            for batch in batch_list:
+                item = (batch.created, batch)
+                queue = self._in_flight_batches[batch.topic_partition]
+                heapq.heappush(queue, item)
+
+        if self.config['guarantee_message_order']:
+            # Mute all the partitions drained
+            for batch_list in six.itervalues(batches_by_node):
+                for batch in batch_list:
+                    self._accumulator.muted.add(batch.topic_partition)
+
+        self._accumulator.reset_next_batch_expiry_time()
+        expired_batches = self._accumulator.expired_batches(now=now)
+        expired_batches.extend(self._get_expired_inflight_batches(now=now))
+
+        if expired_batches:
+            log.debug("%s: Expired %s batches in accumulator", str(self), len(expired_batches))
+
+        # Reset the producer_id if an expired batch has previously been sent to the broker.
+        # See the documentation of `TransactionState.reset_producer_id` to understand why
+        # we need to reset the producer id here.
+        if self._transaction_manager and any([batch.in_retry() for batch in expired_batches]):
+            needs_transaction_state_reset = True
+        else:
+            needs_transaction_state_reset = False
+
+        for expired_batch in expired_batches:
+            error = Errors.KafkaTimeoutError(
+                "Expiring %d record(s) for %s: %s ms has passed since batch creation" % (
+                    expired_batch.record_count, expired_batch.topic_partition,
+                    int((time.time() - expired_batch.created) * 1000)))
+            self._fail_batch(expired_batch, error, base_offset=-1)
+
+        if self._sensors:
+            self._sensors.update_produce_request_metrics(batches_by_node)
+
+        if needs_transaction_state_reset:
+            self._transaction_manager.reset_producer_id()
+            return 0
+
+        requests = self._create_produce_requests(batches_by_node)
+        # If we have any nodes that are ready to send + have sendable data,
+        # poll with 0 timeout so this can immediately loop and try sending more
+        # data. Otherwise, the timeout will be the smaller value between next
+        # batch expiry time, and the delay time for checking data availability.
+        # Note that the nodes may have data that isn't yet sendable due to
+        # lingering, backing off, etc. This specifically does not include nodes with
+        # sendable data that aren't ready to send since they would cause busy
+        # looping.
+        poll_timeout_ms = min(next_ready_check_delay * 1000,
+                              not_ready_timeout_ms,
+                              self._accumulator.next_expiry_time_ms - now * 1000)
+        if poll_timeout_ms < 0:
+            poll_timeout_ms = 0
+
+        if ready_nodes:
+            log.debug("%s: Nodes with data ready to send: %s", str(self), ready_nodes) # trace
+            log.debug("%s: Created %d produce requests: %s", str(self), len(requests), requests) # trace
+            # if some partitions are already ready to be sent, the select time
+            # would be 0; otherwise if some partition already has some data
+            # accumulated but not ready yet, the select time will be the time
+            # difference between now and its linger expiry time; otherwise the
+            # select time will be the time difference between now and the
+            # metadata expiry time
+            poll_timeout_ms = 0
+
+        for node_id, request in six.iteritems(requests):
+            batches = batches_by_node[node_id]
+            log.debug('%s: Sending Produce Request: %r', str(self), request)
+            (self._client.send(node_id, request, wakeup=False)
+                 .add_callback(
+                     self._handle_produce_response, node_id, time.time(), batches)
+                 .add_errback(
+                     self._failed_produce, batches, node_id))
+        return poll_timeout_ms
+
+    def _maybe_send_transactional_request(self):
+        if self._transaction_manager.is_completing() and self._accumulator.has_incomplete:
+            if self._transaction_manager.is_aborting():
+                self._accumulator.abort_undrained_batches(Errors.KafkaError("Failing batch since transaction was aborted"))
+            # There may still be requests left which are being retried. Since we do not know whether they had
+            # been successfully appended to the broker log, we must resend them until their final status is clear.
+            # If they had been appended and we did not receive the error, then our sequence number would no longer
+            # be correct which would lead to an OutOfSequenceNumberError.
+            if not self._accumulator.flush_in_progress():
+                self._accumulator.begin_flush()
+
+        next_request_handler = self._transaction_manager.next_request_handler(self._accumulator.has_incomplete)
+        if next_request_handler is None:
+            return False
+
+        log.debug("%s: Sending transactional request %s", str(self), next_request_handler.request)
+        while not self._force_close:
+            target_node = None
+            try:
+                if next_request_handler.needs_coordinator():
+                    target_node = self._transaction_manager.coordinator(next_request_handler.coordinator_type)
+                    if target_node is None:
+                        self._transaction_manager.lookup_coordinator_for_request(next_request_handler)
+                        break
+                    elif not self._client.await_ready(target_node, timeout_ms=self.config['request_timeout_ms']):
+                        self._transaction_manager.lookup_coordinator_for_request(next_request_handler)
+                        target_node = None
+                        break
+                else:
+                    target_node = self._client.least_loaded_node()
+                    if target_node is not None and not self._client.await_ready(target_node, timeout_ms=self.config['request_timeout_ms']):
+                        target_node = None
+
+                if target_node is not None:
+                    if next_request_handler.is_retry:
+                        time.sleep(self.config['retry_backoff_ms'] / 1000)
+                    txn_correlation_id = self._transaction_manager.next_in_flight_request_correlation_id()
+                    future = self._client.send(target_node, next_request_handler.request)
+                    future.add_both(next_request_handler.on_complete, txn_correlation_id)
+                    return True
+
+            except Exception as e:
+                log.warn("%s: Got an exception when trying to find a node to send a transactional request to. Going to back off and retry: %s", str(self), e)
+                if next_request_handler.needs_coordinator():
+                    self._transaction_manager.lookup_coordinator_for_request(next_request_handler)
+                    break
+
+            time.sleep(self.config['retry_backoff_ms'] / 1000)
+            self._metadata.request_update()
+
+        if target_node is None:
+            self._transaction_manager.retry(next_request_handler)
+
+        return True
+
+    def _maybe_abort_batches(self, exc):
+        if self._accumulator.has_incomplete:
+            log.error("%s: Aborting producer batches due to fatal error: %s", str(self), exc)
+            self._accumulator.abort_batches(exc)
+
+    def initiate_close(self):
+        """Start closing the sender (won't complete until all data is sent)."""
+        self._running = False
+        self._accumulator.close()
+        self.wakeup()
+
+    def force_close(self):
+        """Closes the sender without sending out any pending messages."""
+        self._force_close = True
+        self.initiate_close()
+
+    def add_topic(self, topic):
+        # This is generally called from a separate thread
+        # so this needs to be a thread-safe operation
+        # we assume that checking set membership across threads
+        # is ok where self._client._topics should never
+        # remove topics for a producer instance, only add them.
+        if topic not in self._client._topics:
+            self._topics_to_add.add(topic)
+            self.wakeup()
+
+    def _maybe_wait_for_producer_id(self):
+        while not self._transaction_manager.has_producer_id():
+            try:
+                node_id = self._client.least_loaded_node()
+                if node_id is None or not self._client.await_ready(node_id):
+                    log.debug("%s, Could not find an available broker to send InitProducerIdRequest to." +
+                              " Will back off and try again.", str(self))
+                    time.sleep(self._client.least_loaded_node_refresh_ms() / 1000)
+                    continue
+                version = self._client.api_version(InitProducerIdRequest, max_version=1)
+                request = InitProducerIdRequest[version](
+                    transactional_id=self.config['transactional_id'],
+                    transaction_timeout_ms=self.config['transaction_timeout_ms'],
+                )
+                response = self._client.send_and_receive(node_id, request)
+                error_type = Errors.for_code(response.error_code)
+                if error_type is Errors.NoError:
+                    self._transaction_manager.set_producer_id_and_epoch(ProducerIdAndEpoch(response.producer_id, response.producer_epoch))
+                    break
+                elif getattr(error_type, 'retriable', False):
+                    log.debug("%s: Retriable error from InitProducerId response: %s", str(self), error_type.__name__)
+                    if getattr(error_type, 'invalid_metadata', False):
+                        self._metadata.request_update()
+                else:
+                    self._transaction_manager.transition_to_fatal_error(error_type())
+                    break
+            except Errors.KafkaConnectionError:
+                log.debug("%s: Broker %s disconnected while awaiting InitProducerId response", str(self), node_id)
+            except Errors.RequestTimedOutError:
+                log.debug("%s: InitProducerId request to node %s timed out", str(self), node_id)
+            log.debug("%s: Retry InitProducerIdRequest in %sms.", str(self), self.config['retry_backoff_ms'])
+            time.sleep(self.config['retry_backoff_ms'] / 1000)
+
+    def _failed_produce(self, batches, node_id, error):
+        log.error("%s: Error sending produce request to node %d: %s", str(self), node_id, error) # trace
+        for batch in batches:
+            self._complete_batch(batch, error, -1)
+
+    def _handle_produce_response(self, node_id, send_time, batches, response):
+        """Handle a produce response."""
+        # if we have a response, parse it
+        log.debug('%s: Parsing produce response: %r', str(self), response)
+        if response:
+            batches_by_partition = dict([(batch.topic_partition, batch)
+                                         for batch in batches])
+
+            for topic, partitions in response.topics:
+                for partition_info in partitions:
+                    if response.API_VERSION < 2:
+                        partition, error_code, offset = partition_info
+                        ts = None
+                    elif 2 <= response.API_VERSION <= 4:
+                        partition, error_code, offset, ts = partition_info
+                    elif 5 <= response.API_VERSION <= 7:
+                        partition, error_code, offset, ts, _log_start_offset = partition_info
+                    else:
+                        # Currently unused / TODO: KIP-467
+                        partition, error_code, offset, ts, _log_start_offset, _record_errors, _global_error = partition_info
+                    tp = TopicPartition(topic, partition)
+                    error = Errors.for_code(error_code)
+                    batch = batches_by_partition[tp]
+                    self._complete_batch(batch, error, offset, timestamp_ms=ts)
+
+        else:
+            # this is the acks = 0 case, just complete all requests
+            for batch in batches:
+                self._complete_batch(batch, None, -1)
+
+    def _fail_batch(self, batch, exception, base_offset=None, timestamp_ms=None):
+        exception = exception if type(exception) is not type else exception()
+        if self._transaction_manager:
+            if isinstance(exception, Errors.OutOfOrderSequenceNumberError) and \
+                    not self._transaction_manager.is_transactional() and \
+                    self._transaction_manager.has_producer_id(batch.producer_id):
+                log.error("%s: The broker received an out of order sequence number for topic-partition %s"
+                          " at offset %s. This indicates data loss on the broker, and should be investigated.",
+                          str(self), batch.topic_partition, base_offset)
+
+                # Reset the transaction state since we have hit an irrecoverable exception and cannot make any guarantees
+                # about the previously committed message. Note that this will discard the producer id and sequence
+                # numbers for all existing partitions.
+                self._transaction_manager.reset_producer_id()
+            elif isinstance(exception, (Errors.ClusterAuthorizationFailedError,
+                                        Errors.TransactionalIdAuthorizationFailedError,
+                                        Errors.ProducerFencedError,
+                                        Errors.InvalidTxnStateError)):
+                self._transaction_manager.transition_to_fatal_error(exception)
+            elif self._transaction_manager.is_transactional():
+                self._transaction_manager.transition_to_abortable_error(exception)
+
+        if self._sensors:
+            self._sensors.record_errors(batch.topic_partition.topic, batch.record_count)
+
+        if batch.done(base_offset=base_offset, timestamp_ms=timestamp_ms, exception=exception):
+            self._maybe_remove_from_inflight_batches(batch)
+            self._accumulator.deallocate(batch)
+
+    def _complete_batch(self, batch, error, base_offset, timestamp_ms=None):
+        """Complete or retry the given batch of records.
+
+        Arguments:
+            batch (ProducerBatch): The record batch
+            error (Exception): The error (or None if none)
+            base_offset (int): The base offset assigned to the records if successful
+            timestamp_ms (int, optional): The timestamp returned by the broker for this batch
+        """
+        # Standardize no-error to None
+        if error is Errors.NoError:
+            error = None
+
+        if error is not None:
+            if self._can_retry(batch, error):
+                # retry
+                log.warning("%s: Got error produce response on topic-partition %s,"
+                            " retrying (%s attempts left). Error: %s",
+                            str(self), batch.topic_partition,
+                            self.config['retries'] - batch.attempts - 1,
+                            error)
+
+                # If idempotence is enabled only retry the request if the batch matches our current producer id and epoch
+                if not self._transaction_manager or self._transaction_manager.producer_id_and_epoch.match(batch):
+                    log.debug("%s: Retrying batch to topic-partition %s. Sequence number: %s",
+                              str(self), batch.topic_partition,
+                              self._transaction_manager.sequence_number(batch.topic_partition) if self._transaction_manager else None)
+                    self._accumulator.reenqueue(batch)
+                    self._maybe_remove_from_inflight_batches(batch)
+                    if self._sensors:
+                        self._sensors.record_retries(batch.topic_partition.topic, batch.record_count)
+                else:
+                    log.warning("%s: Attempted to retry sending a batch but the producer id/epoch changed from %s/%s to %s/%s. This batch will be dropped",
+                                str(self), batch.producer_id, batch.producer_epoch,
+                                self._transaction_manager.producer_id_and_epoch.producer_id,
+                                self._transaction_manager.producer_id_and_epoch.epoch)
+                    self._fail_batch(batch, error, base_offset=base_offset, timestamp_ms=timestamp_ms)
+            else:
+                if error is Errors.TopicAuthorizationFailedError:
+                    error = error(batch.topic_partition.topic)
+
+                # tell the user the result of their request
+                self._fail_batch(batch, error, base_offset=base_offset, timestamp_ms=timestamp_ms)
+
+            if error is Errors.UnknownTopicOrPartitionError:
+                log.warning("%s: Received unknown topic or partition error in produce request on partition %s."
+                            " The topic/partition may not exist or the user may not have Describe access to it",
+                            str(self), batch.topic_partition)
+
+            if getattr(error, 'invalid_metadata', False):
+                self._metadata.request_update()
+
+        else:
+            if batch.done(base_offset=base_offset, timestamp_ms=timestamp_ms):
+                self._maybe_remove_from_inflight_batches(batch)
+                self._accumulator.deallocate(batch)
+
+            if self._transaction_manager and self._transaction_manager.producer_id_and_epoch.match(batch):
+                self._transaction_manager.increment_sequence_number(batch.topic_partition, batch.record_count)
+                log.debug("%s: Incremented sequence number for topic-partition %s to %s", str(self), batch.topic_partition,
+                          self._transaction_manager.sequence_number(batch.topic_partition))
+
+        # Unmute the completed partition.
+        if self.config['guarantee_message_order']:
+            self._accumulator.muted.remove(batch.topic_partition)
+
+    def _can_retry(self, batch, error):
+        """
+        We can retry a send if the error is transient and the number of
+        attempts taken is fewer than the maximum allowed
+        """
+        return (not batch.has_reached_delivery_timeout(self._accumulator.delivery_timeout_ms) and
+                batch.attempts < self.config['retries'] and
+                batch.final_state is None and
+                getattr(error, 'retriable', False))
+
+    def _create_produce_requests(self, collated):
+        """
+        Transfer the record batches into a list of produce requests on a
+        per-node basis.
+
+        Arguments:
+            collated: {node_id: [ProducerBatch]}
+
+        Returns:
+            dict: {node_id: ProduceRequest} (version depends on client api_versions)
+        """
+        requests = {}
+        for node_id, batches in six.iteritems(collated):
+            if batches:
+                requests[node_id] = self._produce_request(
+                    node_id, self.config['acks'],
+                    self.config['request_timeout_ms'], batches)
+        return requests
+
+    def _produce_request(self, node_id, acks, timeout, batches):
+        """Create a produce request from the given record batches.
+
+        Returns:
+            ProduceRequest (version depends on client api_versions)
+        """
+        produce_records_by_partition = collections.defaultdict(dict)
+        for batch in batches:
+            topic = batch.topic_partition.topic
+            partition = batch.topic_partition.partition
+
+            buf = batch.records.buffer()
+            produce_records_by_partition[topic][partition] = buf
+
+        version = self._client.api_version(ProduceRequest, max_version=7)
+        topic_partition_data = [
+            (topic, list(partition_info.items()))
+            for topic, partition_info in six.iteritems(produce_records_by_partition)]
+        transactional_id = self._transaction_manager.transactional_id if self._transaction_manager else None
+        if version >= 3:
+            return ProduceRequest[version](
+                transactional_id=transactional_id,
+                required_acks=acks,
+                timeout=timeout,
+                topics=topic_partition_data,
+            )
+        else:
+            if transactional_id is not None:
+                log.warning('%s: Broker does not support ProduceRequest v3+, required for transactional_id', str(self))
+            return ProduceRequest[version](
+                required_acks=acks,
+                timeout=timeout,
+                topics=topic_partition_data,
+            )
+
+    def wakeup(self):
+        """Wake up the selector associated with this send thread."""
+        self._client.wakeup()
+
+    def bootstrap_connected(self):
+        return self._client.bootstrap_connected()
+
+    def __str__(self):
+        return "<Sender client_id=%s transactional_id=%s>" % (self.config['client_id'], self.config['transactional_id'])
+
+
+class SenderMetrics(object):
+
+    def __init__(self, metrics, client, metadata):
+        self.metrics = metrics
+        self._client = client
+        self._metadata = metadata
+
+        sensor_name = 'batch-size'
+        self.batch_size_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('batch-size-avg', Avg(),
+                        sensor_name=sensor_name,
+                        description='The average number of bytes sent per partition per-request.')
+        self.add_metric('batch-size-max', Max(),
+                        sensor_name=sensor_name,
+                        description='The max number of bytes sent per partition per-request.')
+
+        sensor_name = 'compression-rate'
+        self.compression_rate_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('compression-rate-avg', Avg(),
+                        sensor_name=sensor_name,
+                        description='The average compression rate of record batches.')
+
+        sensor_name = 'queue-time'
+        self.queue_time_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('record-queue-time-avg', Avg(),
+                        sensor_name=sensor_name,
+                        description='The average time in ms record batches spent in the record accumulator.')
+        self.add_metric('record-queue-time-max', Max(),
+                        sensor_name=sensor_name,
+                        description='The maximum time in ms record batches spent in the record accumulator.')
+
+        sensor_name = 'records-per-request'
+        self.records_per_request_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('record-send-rate', Rate(),
+                        sensor_name=sensor_name,
+                        description='The average number of records sent per second.')
+        self.add_metric('records-per-request-avg', Avg(),
+                        sensor_name=sensor_name,
+                        description='The average number of records per request.')
+
+        sensor_name = 'bytes'
+        self.byte_rate_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('byte-rate', Rate(),
+                        sensor_name=sensor_name,
+                        description='The average number of bytes sent per second.')
+
+        sensor_name = 'record-retries'
+        self.retry_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('record-retry-rate', Rate(),
+                        sensor_name=sensor_name,
+                        description='The average per-second number of retried record sends')
+
+        sensor_name = 'errors'
+        self.error_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('record-error-rate', Rate(),
+                        sensor_name=sensor_name,
+                        description='The average per-second number of record sends that resulted in errors')
+
+        sensor_name = 'record-size-max'
+        self.max_record_size_sensor = self.metrics.sensor(sensor_name)
+        self.add_metric('record-size-max', Max(),
+                        sensor_name=sensor_name,
+                        description='The maximum record size across all batches')
+        self.add_metric('record-size-avg', Avg(),
+                        sensor_name=sensor_name,
+                        description='The average maximum record size per batch')
+
+        self.add_metric('requests-in-flight',
+                        AnonMeasurable(lambda *_: self._client.in_flight_request_count()),
+                        description='The current number of in-flight requests awaiting a response.')
+
+        self.add_metric('metadata-age',
+                        AnonMeasurable(lambda _, now: (now - self._metadata._last_successful_refresh_ms) / 1000),
+                        description='The age in seconds of the current producer metadata being used.')
+
+    def add_metric(self, metric_name, measurable, group_name='producer-metrics',
+                   description=None, tags=None,
+                   sensor_name=None):
+        m = self.metrics
+        metric = m.metric_name(metric_name, group_name, description, tags)
+        if sensor_name:
+            sensor = m.sensor(sensor_name)
+            sensor.add(metric, measurable)
+        else:
+            m.add_metric(metric, measurable)
+
+    def maybe_register_topic_metrics(self, topic):
+
+        def sensor_name(name):
+            return 'topic.{0}.{1}'.format(topic, name)
+
+        # if one sensor of the metrics has been registered for the topic,
+        # then all other sensors should have been registered; and vice versa
+        if not self.metrics.get_sensor(sensor_name('records-per-batch')):
+
+            self.add_metric('record-send-rate', Rate(),
+                            sensor_name=sensor_name('records-per-batch'),
+                            group_name='producer-topic-metrics.' + topic,
+                            description= 'Records sent per second for topic ' + topic)
+
+            self.add_metric('byte-rate', Rate(),
+                            sensor_name=sensor_name('bytes'),
+                            group_name='producer-topic-metrics.' + topic,
+                            description='Bytes per second for topic ' + topic)
+
+            self.add_metric('compression-rate', Avg(),
+                            sensor_name=sensor_name('compression-rate'),
+                            group_name='producer-topic-metrics.' + topic,
+                            description='Average Compression ratio for topic ' + topic)
+
+            self.add_metric('record-retry-rate', Rate(),
+                            sensor_name=sensor_name('record-retries'),
+                            group_name='producer-topic-metrics.' + topic,
+                            description='Record retries per second for topic ' + topic)
+
+            self.add_metric('record-error-rate', Rate(),
+                            sensor_name=sensor_name('record-errors'),
+                            group_name='producer-topic-metrics.' + topic,
+                            description='Record errors per second for topic ' + topic)
+
+    def update_produce_request_metrics(self, batches_map):
+        for node_batch in batches_map.values():
+            records = 0
+            total_bytes = 0
+            for batch in node_batch:
+                # register all per-topic metrics at once
+                topic = batch.topic_partition.topic
+                self.maybe_register_topic_metrics(topic)
+
+                # per-topic record send rate
+                topic_records_count = self.metrics.get_sensor(
+                    'topic.' + topic + '.records-per-batch')
+                topic_records_count.record(batch.record_count)
+
+                # per-topic bytes send rate
+                topic_byte_rate = self.metrics.get_sensor(
+                    'topic.' + topic + '.bytes')
+                topic_byte_rate.record(batch.records.size_in_bytes())
+
+                # per-topic compression rate
+                topic_compression_rate = self.metrics.get_sensor(
+                    'topic.' + topic + '.compression-rate')
+                topic_compression_rate.record(batch.records.compression_rate())
+
+                # global metrics
+                self.batch_size_sensor.record(batch.records.size_in_bytes())
+                if batch.drained:
+                    self.queue_time_sensor.record(batch.drained - batch.created)
+                self.compression_rate_sensor.record(batch.records.compression_rate())
+                self.max_record_size_sensor.record(batch.max_record_size)
+                records += batch.record_count
+                total_bytes += batch.records.size_in_bytes()
+
+            if node_batch:
+                self.records_per_request_sensor.record(records)
+                self.byte_rate_sensor.record(total_bytes)
+
+    def record_retries(self, topic, count):
+        self.retry_sensor.record(count)
+        sensor = self.metrics.get_sensor('topic.' + topic + '.record-retries')
+        if sensor:
+            sensor.record(count)
+
+    def record_errors(self, topic, count):
+        self.error_sensor.record(count)
+        sensor = self.metrics.get_sensor('topic.' + topic + '.record-errors')
+        if sensor:
+            sensor.record(count)

venv/lib/python3.11/site-packages/kafka/producer/transaction_manager.py

@@ -0,0 +1,981 @@
+from __future__ import absolute_import, division
+
+import abc 
+import collections
+import heapq
+import logging
+import threading
+
+from kafka.vendor import six
+
+try:
+    # enum in stdlib as of py3.4
+    from enum import IntEnum  # pylint: disable=import-error
+except ImportError:
+    # vendored backport module
+    from kafka.vendor.enum34 import IntEnum
+
+import kafka.errors as Errors
+from kafka.protocol.add_offsets_to_txn import AddOffsetsToTxnRequest
+from kafka.protocol.add_partitions_to_txn import AddPartitionsToTxnRequest
+from kafka.protocol.end_txn import EndTxnRequest
+from kafka.protocol.find_coordinator import FindCoordinatorRequest
+from kafka.protocol.init_producer_id import InitProducerIdRequest
+from kafka.protocol.txn_offset_commit import TxnOffsetCommitRequest
+from kafka.structs import TopicPartition
+
+
+log = logging.getLogger(__name__)
+
+
+NO_PRODUCER_ID = -1
+NO_PRODUCER_EPOCH = -1
+NO_SEQUENCE = -1
+
+
+class ProducerIdAndEpoch(object):
+    __slots__ = ('producer_id', 'epoch')
+
+    def __init__(self, producer_id, epoch):
+        self.producer_id = producer_id
+        self.epoch = epoch
+
+    @property
+    def is_valid(self):
+        return NO_PRODUCER_ID < self.producer_id
+
+    def match(self, batch):
+        return self.producer_id == batch.producer_id and self.epoch == batch.producer_epoch
+
+    def __eq__(self, other):
+        return isinstance(other, ProducerIdAndEpoch) and self.producer_id == other.producer_id and self.epoch == other.epoch
+
+    def __str__(self):
+        return "ProducerIdAndEpoch(producer_id={}, epoch={})".format(self.producer_id, self.epoch)
+
+
+class TransactionState(IntEnum):
+    UNINITIALIZED = 0
+    INITIALIZING = 1
+    READY = 2
+    IN_TRANSACTION = 3
+    COMMITTING_TRANSACTION = 4
+    ABORTING_TRANSACTION = 5
+    ABORTABLE_ERROR = 6
+    FATAL_ERROR = 7
+
+    @classmethod
+    def is_transition_valid(cls, source, target):
+        if target == cls.INITIALIZING:
+            return source == cls.UNINITIALIZED
+        elif target == cls.READY:
+            return source in (cls.INITIALIZING, cls.COMMITTING_TRANSACTION, cls.ABORTING_TRANSACTION)
+        elif target == cls.IN_TRANSACTION:
+            return source == cls.READY
+        elif target == cls.COMMITTING_TRANSACTION:
+            return source == cls.IN_TRANSACTION
+        elif target == cls.ABORTING_TRANSACTION:
+            return source in (cls.IN_TRANSACTION, cls.ABORTABLE_ERROR)
+        elif target == cls.ABORTABLE_ERROR:
+            return source in (cls.IN_TRANSACTION, cls.COMMITTING_TRANSACTION, cls.ABORTABLE_ERROR)
+        elif target == cls.UNINITIALIZED:
+            # Disallow transitions to UNITIALIZED
+            return False
+        elif target == cls.FATAL_ERROR:
+            # We can transition to FATAL_ERROR unconditionally.
+            # FATAL_ERROR is never a valid starting state for any transition. So the only option is to close the
+            # producer or do purely non transactional requests.
+            return True
+
+
+class Priority(IntEnum):
+    # We use the priority to determine the order in which requests need to be sent out. For instance, if we have
+    # a pending FindCoordinator request, that must always go first. Next, If we need a producer id, that must go second.
+    # The endTxn request must always go last.
+    FIND_COORDINATOR = 0
+    INIT_PRODUCER_ID = 1
+    ADD_PARTITIONS_OR_OFFSETS = 2
+    END_TXN = 3
+
+
+class TransactionManager(object):
+    """
+    A class which maintains state for transactions. Also keeps the state necessary to ensure idempotent production.
+    """
+    NO_INFLIGHT_REQUEST_CORRELATION_ID = -1
+    # The retry_backoff_ms is overridden to the following value if the first AddPartitions receives a
+    # CONCURRENT_TRANSACTIONS error.
+    ADD_PARTITIONS_RETRY_BACKOFF_MS = 20
+
+    def __init__(self, transactional_id=None, transaction_timeout_ms=0, retry_backoff_ms=100, api_version=(0, 11), metadata=None):
+        self._api_version = api_version
+        self._metadata = metadata
+
+        self._sequence_numbers = collections.defaultdict(lambda: 0)
+
+        self.transactional_id = transactional_id
+        self.transaction_timeout_ms = transaction_timeout_ms
+        self._transaction_coordinator = None
+        self._consumer_group_coordinator = None
+        self._new_partitions_in_transaction = set()
+        self._pending_partitions_in_transaction = set()
+        self._partitions_in_transaction = set()
+        self._pending_txn_offset_commits = dict()
+
+        self._current_state = TransactionState.UNINITIALIZED
+        self._last_error = None
+        self.producer_id_and_epoch = ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH)
+
+        self._transaction_started = False
+
+        self._pending_requests = [] # priority queue via heapq
+        self._pending_requests_sort_id = 0
+        self._in_flight_request_correlation_id = self.NO_INFLIGHT_REQUEST_CORRELATION_ID
+
+        # This is used by the TxnRequestHandlers to control how long to back off before a given request is retried.
+        # For instance, this value is lowered by the AddPartitionsToTxnHandler when it receives a CONCURRENT_TRANSACTIONS
+        # error for the first AddPartitionsRequest in a transaction.
+        self.retry_backoff_ms = retry_backoff_ms
+        self._lock = threading.Condition()
+
+    def initialize_transactions(self):
+        with self._lock:
+            self._ensure_transactional()
+            self._transition_to(TransactionState.INITIALIZING)
+            self.set_producer_id_and_epoch(ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH))
+            self._sequence_numbers.clear()
+            handler = InitProducerIdHandler(self, self.transaction_timeout_ms)
+            self._enqueue_request(handler)
+            return handler.result
+
+    def begin_transaction(self):
+        with self._lock:
+            self._ensure_transactional()
+            self._maybe_fail_with_error()
+            self._transition_to(TransactionState.IN_TRANSACTION)
+
+    def begin_commit(self):
+        with self._lock:
+            self._ensure_transactional()
+            self._maybe_fail_with_error()
+            self._transition_to(TransactionState.COMMITTING_TRANSACTION)
+            return self._begin_completing_transaction(True)
+
+    def begin_abort(self):
+        with self._lock:
+            self._ensure_transactional()
+            if self._current_state != TransactionState.ABORTABLE_ERROR:
+                self._maybe_fail_with_error()
+            self._transition_to(TransactionState.ABORTING_TRANSACTION)
+
+            # We're aborting the transaction, so there should be no need to add new partitions
+            self._new_partitions_in_transaction.clear()
+            return self._begin_completing_transaction(False)
+
+    def _begin_completing_transaction(self, committed):
+        if self._new_partitions_in_transaction:
+            self._enqueue_request(self._add_partitions_to_transaction_handler())
+        handler = EndTxnHandler(self, committed)
+        self._enqueue_request(handler)
+        return handler.result
+
+    def send_offsets_to_transaction(self, offsets, consumer_group_id):
+        with self._lock:
+            self._ensure_transactional()
+            self._maybe_fail_with_error()
+            if self._current_state != TransactionState.IN_TRANSACTION:
+                raise Errors.KafkaError("Cannot send offsets to transaction because the producer is not in an active transaction")
+
+            log.debug("Begin adding offsets %s for consumer group %s to transaction", offsets, consumer_group_id)
+            handler = AddOffsetsToTxnHandler(self, consumer_group_id, offsets)
+            self._enqueue_request(handler)
+            return handler.result
+
+    def maybe_add_partition_to_transaction(self, topic_partition):
+        with self._lock:
+            self._fail_if_not_ready_for_send()
+
+            if self.is_partition_added(topic_partition) or self.is_partition_pending_add(topic_partition):
+                return
+
+            log.debug("Begin adding new partition %s to transaction", topic_partition)
+            self._new_partitions_in_transaction.add(topic_partition)
+
+    def _fail_if_not_ready_for_send(self):
+        with self._lock:
+            if self.has_error():
+                raise Errors.KafkaError(
+                        "Cannot perform send because at least one previous transactional or"
+                        " idempotent request has failed with errors.", self._last_error)
+
+            if self.is_transactional():
+                if not self.has_producer_id():
+                    raise Errors.IllegalStateError(
+                            "Cannot perform a 'send' before completing a call to init_transactions"
+                            " when transactions are enabled.")
+
+                if self._current_state != TransactionState.IN_TRANSACTION:
+                    raise Errors.IllegalStateError("Cannot call send in state %s" % (self._current_state.name,))
+
+    def is_send_to_partition_allowed(self, tp):
+        with self._lock:
+            if self.has_fatal_error():
+                return False
+            return not self.is_transactional() or tp in self._partitions_in_transaction
+
+    def has_producer_id(self, producer_id=None):
+        if producer_id is None:
+            return self.producer_id_and_epoch.is_valid
+        else:
+            return self.producer_id_and_epoch.producer_id == producer_id
+
+    def is_transactional(self):
+        return self.transactional_id is not None
+
+    def has_partitions_to_add(self):
+        with self._lock:
+            return bool(self._new_partitions_in_transaction) or bool(self._pending_partitions_in_transaction)
+
+    def is_completing(self):
+        with self._lock:
+            return self._current_state in (
+                TransactionState.COMMITTING_TRANSACTION,
+                TransactionState.ABORTING_TRANSACTION)
+
+    @property
+    def last_error(self):
+        return self._last_error
+
+    def has_error(self):
+        with self._lock:
+            return self._current_state in (
+                TransactionState.ABORTABLE_ERROR,
+                TransactionState.FATAL_ERROR)
+
+    def is_aborting(self):
+        with self._lock:
+            return self._current_state == TransactionState.ABORTING_TRANSACTION
+
+    def transition_to_abortable_error(self, exc):
+        with self._lock:
+            if self._current_state == TransactionState.ABORTING_TRANSACTION:
+                log.debug("Skipping transition to abortable error state since the transaction is already being "
+                          " aborted. Underlying exception: %s", exc)
+                return
+            self._transition_to(TransactionState.ABORTABLE_ERROR, error=exc)
+
+    def transition_to_fatal_error(self, exc):
+        with self._lock:
+            self._transition_to(TransactionState.FATAL_ERROR, error=exc)
+
+    def is_partition_added(self, partition):
+        with self._lock:
+            return partition in self._partitions_in_transaction
+
+    def is_partition_pending_add(self, partition):
+        return partition in self._new_partitions_in_transaction or partition in self._pending_partitions_in_transaction
+
+    def has_producer_id_and_epoch(self, producer_id, producer_epoch):
+        return (
+            self.producer_id_and_epoch.producer_id == producer_id and
+            self.producer_id_and_epoch.epoch == producer_epoch
+        )
+
+    def set_producer_id_and_epoch(self, producer_id_and_epoch):
+        if not isinstance(producer_id_and_epoch, ProducerIdAndEpoch):
+            raise TypeError("ProducerAndIdEpoch type required")
+        log.info("ProducerId set to %s with epoch %s",
+                 producer_id_and_epoch.producer_id, producer_id_and_epoch.epoch)
+        self.producer_id_and_epoch = producer_id_and_epoch
+
+    def reset_producer_id(self):
+        """
+        This method is used when the producer needs to reset its internal state because of an irrecoverable exception
+        from the broker.
+
+        We need to reset the producer id and associated state when we have sent a batch to the broker, but we either get
+        a non-retriable exception or we run out of retries, or the batch expired in the producer queue after it was already
+        sent to the broker.
+
+        In all of these cases, we don't know whether batch was actually committed on the broker, and hence whether the
+        sequence number was actually updated. If we don't reset the producer state, we risk the chance that all future
+        messages will return an OutOfOrderSequenceNumberError.
+
+        Note that we can't reset the producer state for the transactional producer as this would mean bumping the epoch
+        for the same producer id. This might involve aborting the ongoing transaction during the initProducerIdRequest,
+        and the user would not have any way of knowing this happened. So for the transactional producer,
+        it's best to return the produce error to the user and let them abort the transaction and close the producer explicitly.
+        """
+        with self._lock:
+            if self.is_transactional():
+                raise Errors.IllegalStateError( 
+                    "Cannot reset producer state for a transactional producer."
+                    " You must either abort the ongoing transaction or"
+                    " reinitialize the transactional producer instead")
+            self.set_producer_id_and_epoch(ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH))
+            self._sequence_numbers.clear()
+
+    def sequence_number(self, tp):
+        with self._lock:
+            return self._sequence_numbers[tp]
+
+    def increment_sequence_number(self, tp, increment):
+        with self._lock:
+            if tp not in self._sequence_numbers:
+                raise Errors.IllegalStateError("Attempt to increment sequence number for a partition with no current sequence.")
+            # Sequence number wraps at java max int
+            base = self._sequence_numbers[tp]
+            if base > (2147483647 - increment):
+              self._sequence_numbers[tp] = increment - (2147483647 - base) - 1
+            else:
+                self._sequence_numbers[tp] += increment
+
+    def next_request_handler(self, has_incomplete_batches):
+        with self._lock:
+            if self._new_partitions_in_transaction:
+                self._enqueue_request(self._add_partitions_to_transaction_handler())
+
+            if not self._pending_requests:
+                return None
+
+            _, _, next_request_handler = self._pending_requests[0]
+            # Do not send the EndTxn until all batches have been flushed
+            if isinstance(next_request_handler, EndTxnHandler) and has_incomplete_batches:
+                return None
+
+            heapq.heappop(self._pending_requests)
+            if self._maybe_terminate_request_with_error(next_request_handler):
+                log.debug("Not sending transactional request %s because we are in an error state",
+                          next_request_handler.request)
+                return None
+
+            if isinstance(next_request_handler, EndTxnHandler) and not self._transaction_started:
+                next_request_handler.result.done()
+                if self._current_state != TransactionState.FATAL_ERROR:
+                    log.debug("Not sending EndTxn for completed transaction since no partitions"
+                              " or offsets were successfully added")
+                    self._complete_transaction()
+                try:
+                    _, _, next_request_handler = heapq.heappop(self._pending_requests)
+                except IndexError:
+                    next_request_handler = None
+
+            if next_request_handler:
+                log.debug("Request %s dequeued for sending", next_request_handler.request)
+
+            return next_request_handler
+
+    def retry(self, request):
+        with self._lock:
+            request.set_retry()
+            self._enqueue_request(request)
+
+    def authentication_failed(self, exc):
+        with self._lock:
+            for _, _, request in self._pending_requests:
+                request.fatal_error(exc)
+
+    def coordinator(self, coord_type):
+        if coord_type == 'group':
+            return self._consumer_group_coordinator
+        elif coord_type == 'transaction':
+            return self._transaction_coordinator
+        else:
+            raise Errors.IllegalStateError("Received an invalid coordinator type: %s" % (coord_type,))
+
+    def lookup_coordinator_for_request(self, request):
+        self._lookup_coordinator(request.coordinator_type, request.coordinator_key)
+
+    def next_in_flight_request_correlation_id(self):
+        self._in_flight_request_correlation_id += 1
+        return self._in_flight_request_correlation_id
+
+    def clear_in_flight_transactional_request_correlation_id(self):
+        self._in_flight_request_correlation_id = self.NO_INFLIGHT_REQUEST_CORRELATION_ID
+
+    def has_in_flight_transactional_request(self):
+        return self._in_flight_request_correlation_id != self.NO_INFLIGHT_REQUEST_CORRELATION_ID
+
+    def has_fatal_error(self):
+        return self._current_state == TransactionState.FATAL_ERROR
+
+    def has_abortable_error(self):
+        return self._current_state == TransactionState.ABORTABLE_ERROR
+
+    # visible for testing
+    def _test_transaction_contains_partition(self, tp):
+        with self._lock:
+            return tp in self._partitions_in_transaction
+
+    # visible for testing
+    def _test_has_pending_offset_commits(self):
+        return bool(self._pending_txn_offset_commits)
+
+    # visible for testing
+    def _test_has_ongoing_transaction(self):
+        with self._lock:
+            # transactions are considered ongoing once started until completion or a fatal error
+            return self._current_state == TransactionState.IN_TRANSACTION or self.is_completing() or self.has_abortable_error()
+
+    # visible for testing
+    def _test_is_ready(self):
+        with self._lock:
+            return self.is_transactional() and self._current_state == TransactionState.READY
+
+    def _transition_to(self, target, error=None):
+        with self._lock:
+            if not self._current_state.is_transition_valid(self._current_state, target):
+                raise Errors.KafkaError("TransactionalId %s: Invalid transition attempted from state %s to state %s" % (
+                    self.transactional_id, self._current_state.name, target.name))
+
+            if target in (TransactionState.FATAL_ERROR, TransactionState.ABORTABLE_ERROR):
+                if error is None:
+                    raise Errors.IllegalArgumentError("Cannot transition to %s with an None exception" % (target.name,))
+                self._last_error = error
+            else:
+                self._last_error = None
+
+            if self._last_error is not None:
+                log.debug("Transition from state %s to error state %s (%s)", self._current_state.name, target.name, self._last_error)
+            else:
+                log.debug("Transition from state %s to %s", self._current_state, target)
+            self._current_state = target
+
+    def _ensure_transactional(self):
+        if not self.is_transactional():
+            raise Errors.IllegalStateError("Transactional method invoked on a non-transactional producer.")
+
+    def _maybe_fail_with_error(self):
+        if self.has_error():
+            raise Errors.KafkaError("Cannot execute transactional method because we are in an error state: %s" % (self._last_error,))
+
+    def _maybe_terminate_request_with_error(self, request_handler):
+        if self.has_error():
+            if self.has_abortable_error() and isinstance(request_handler, FindCoordinatorHandler):
+                # No harm letting the FindCoordinator request go through if we're expecting to abort
+                return False
+            request_handler.fail(self._last_error)
+            return True
+        return False
+
+    def _next_pending_requests_sort_id(self):
+        self._pending_requests_sort_id += 1
+        return self._pending_requests_sort_id
+
+    def _enqueue_request(self, request_handler):
+        log.debug("Enqueuing transactional request %s", request_handler.request)
+        heapq.heappush(
+            self._pending_requests,
+            (
+                request_handler.priority, # keep lowest priority at head of queue
+                self._next_pending_requests_sort_id(), # break ties
+                request_handler
+            )
+        )
+
+    def _lookup_coordinator(self, coord_type, coord_key):
+        with self._lock:
+            if coord_type == 'group':
+                self._consumer_group_coordinator = None
+            elif coord_type == 'transaction':
+                self._transaction_coordinator = None
+            else:
+                raise Errors.IllegalStateError("Invalid coordinator type: %s" % (coord_type,))
+        self._enqueue_request(FindCoordinatorHandler(self, coord_type, coord_key))
+
+    def _complete_transaction(self):
+        with self._lock:
+            self._transition_to(TransactionState.READY)
+            self._transaction_started = False
+            self._new_partitions_in_transaction.clear()
+            self._pending_partitions_in_transaction.clear()
+            self._partitions_in_transaction.clear()
+
+    def _add_partitions_to_transaction_handler(self):
+        with self._lock:
+            self._pending_partitions_in_transaction.update(self._new_partitions_in_transaction)
+            self._new_partitions_in_transaction.clear()
+            return AddPartitionsToTxnHandler(self, self._pending_partitions_in_transaction)
+
+
+class TransactionalRequestResult(object):
+    def __init__(self):
+        self._latch = threading.Event()
+        self._error = None
+
+    def done(self, error=None):
+        self._error = error
+        self._latch.set()
+
+    def wait(self, timeout_ms=None):
+        timeout = timeout_ms / 1000 if timeout_ms is not None else None
+        success = self._latch.wait(timeout)
+        if self._error:
+            raise self._error
+        return success
+
+    @property
+    def is_done(self):
+        return self._latch.is_set()
+
+    @property
+    def succeeded(self):
+        return self._latch.is_set() and self._error is None
+
+    @property
+    def failed(self):
+        return self._latch.is_set() and self._error is not None
+
+    @property
+    def exception(self):
+        return self._error
+
+
+@six.add_metaclass(abc.ABCMeta)
+class TxnRequestHandler(object):
+    def __init__(self, transaction_manager, result=None):
+        self.transaction_manager = transaction_manager
+        self.retry_backoff_ms = transaction_manager.retry_backoff_ms
+        self.request = None
+        self._result = result or TransactionalRequestResult()
+        self._is_retry = False
+
+    @property
+    def transactional_id(self):
+        return self.transaction_manager.transactional_id
+
+    @property
+    def producer_id(self):
+        return self.transaction_manager.producer_id_and_epoch.producer_id
+
+    @property
+    def producer_epoch(self):
+        return self.transaction_manager.producer_id_and_epoch.epoch
+
+    def fatal_error(self, exc):
+        self.transaction_manager.transition_to_fatal_error(exc)
+        self._result.done(error=exc)
+
+    def abortable_error(self, exc):
+        self.transaction_manager.transition_to_abortable_error(exc)
+        self._result.done(error=exc)
+
+    def fail(self, exc):
+        self._result.done(error=exc)
+
+    def reenqueue(self):
+        with self.transaction_manager._lock:
+            self._is_retry = True
+            self.transaction_manager._enqueue_request(self)
+
+    def on_complete(self, correlation_id, response_or_exc):
+        if correlation_id != self.transaction_manager._in_flight_request_correlation_id:
+            self.fatal_error(RuntimeError("Detected more than one in-flight transactional request."))
+        else:
+            self.transaction_manager.clear_in_flight_transactional_request_correlation_id()
+            if isinstance(response_or_exc, Errors.KafkaConnectionError):
+                log.debug("Disconnected from node. Will retry.")
+                if self.needs_coordinator():
+                    self.transaction_manager._lookup_coordinator(self.coordinator_type, self.coordinator_key)
+                self.reenqueue()
+            elif isinstance(response_or_exc, Errors.UnsupportedVersionError):
+                self.fatal_error(response_or_exc)
+            elif not isinstance(response_or_exc, (Exception, type(None))):
+                log.debug("Received transactional response %s for request %s", response_or_exc, self.request)
+                with self.transaction_manager._lock:
+                    self.handle_response(response_or_exc)
+            else:
+                self.fatal_error(Errors.KafkaError("Could not execute transactional request for unknown reasons: %s" % response_or_exc))
+
+    def needs_coordinator(self):
+        return self.coordinator_type is not None
+
+    @property
+    def result(self):
+        return self._result
+
+    @property
+    def coordinator_type(self):
+        return 'transaction'
+
+    @property
+    def coordinator_key(self):
+        return self.transaction_manager.transactional_id
+
+    def set_retry(self):
+        self._is_retry = True
+
+    @property
+    def is_retry(self):
+        return self._is_retry
+
+    @abc.abstractmethod
+    def handle_response(self, response):
+        pass
+
+    @abc.abstractproperty
+    def priority(self):
+        pass
+
+
+class InitProducerIdHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, transaction_timeout_ms):
+        super(InitProducerIdHandler, self).__init__(transaction_manager)
+
+        if transaction_manager._api_version >= (2, 0):
+            version = 1
+        else:
+            version = 0
+        self.request = InitProducerIdRequest[version](
+            transactional_id=self.transactional_id,
+            transaction_timeout_ms=transaction_timeout_ms)
+
+    @property
+    def priority(self):
+        return Priority.INIT_PRODUCER_ID
+
+    def handle_response(self, response):
+        error = Errors.for_code(response.error_code)
+
+        if error is Errors.NoError:
+            self.transaction_manager.set_producer_id_and_epoch(ProducerIdAndEpoch(response.producer_id, response.producer_epoch))
+            self.transaction_manager._transition_to(TransactionState.READY)
+            self._result.done()
+        elif error in (Errors.NotCoordinatorError, Errors.CoordinatorNotAvailableError):
+            self.transaction_manager._lookup_coordinator('transaction', self.transactional_id)
+            self.reenqueue()
+        elif error in (Errors.CoordinatorLoadInProgressError, Errors.ConcurrentTransactionsError):
+            self.reenqueue()
+        elif error is Errors.TransactionalIdAuthorizationFailedError:
+            self.fatal_error(error())
+        else:
+            self.fatal_error(Errors.KafkaError("Unexpected error in InitProducerIdResponse: %s" % (error())))
+
+class AddPartitionsToTxnHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, topic_partitions):
+        super(AddPartitionsToTxnHandler, self).__init__(transaction_manager)
+
+        if transaction_manager._api_version >= (2, 7):
+            version = 2
+        elif transaction_manager._api_version >= (2, 0):
+            version = 1
+        else:
+            version = 0
+        topic_data = collections.defaultdict(list)
+        for tp in topic_partitions:
+            topic_data[tp.topic].append(tp.partition)
+        self.request = AddPartitionsToTxnRequest[version](
+            transactional_id=self.transactional_id,
+            producer_id=self.producer_id,
+            producer_epoch=self.producer_epoch,
+            topics=list(topic_data.items()))
+
+    @property
+    def priority(self):
+        return Priority.ADD_PARTITIONS_OR_OFFSETS
+
+    def handle_response(self, response):
+        has_partition_errors = False
+        unauthorized_topics = set()
+        self.retry_backoff_ms = self.transaction_manager.retry_backoff_ms
+
+        results = {TopicPartition(topic, partition): Errors.for_code(error_code)
+                   for topic, partition_data in response.results
+                   for partition, error_code in partition_data}
+
+        for tp, error in six.iteritems(results):
+            if error is Errors.NoError:
+                continue
+            elif error in (Errors.CoordinatorNotAvailableError, Errors.NotCoordinatorError):
+                self.transaction_manager._lookup_coordinator('transaction', self.transactional_id)
+                self.reenqueue()
+                return
+            elif error is Errors.ConcurrentTransactionsError:
+                self.maybe_override_retry_backoff_ms()
+                self.reenqueue()
+                return
+            elif error in (Errors.CoordinatorLoadInProgressError, Errors.UnknownTopicOrPartitionError):
+                self.reenqueue()
+                return
+            elif error is Errors.InvalidProducerEpochError:
+                self.fatal_error(error())
+                return
+            elif error is Errors.TransactionalIdAuthorizationFailedError:
+                self.fatal_error(error())
+                return
+            elif error in (Errors.InvalidProducerIdMappingError, Errors.InvalidTxnStateError):
+                self.fatal_error(Errors.KafkaError(error()))
+                return
+            elif error is Errors.TopicAuthorizationFailedError:
+                unauthorized_topics.add(tp.topic)
+            elif error is Errors.OperationNotAttemptedError:
+                log.debug("Did not attempt to add partition %s to transaction because other partitions in the"
+                          " batch had errors.", tp)
+                has_partition_errors = True
+            else:
+                log.error("Could not add partition %s due to unexpected error %s", tp, error())
+                has_partition_errors = True
+
+        partitions = set(results)
+
+        # Remove the partitions from the pending set regardless of the result. We use the presence
+        # of partitions in the pending set to know when it is not safe to send batches. However, if
+        # the partitions failed to be added and we enter an error state, we expect the batches to be
+        # aborted anyway. In this case, we must be able to continue sending the batches which are in
+        # retry for partitions that were successfully added.
+        self.transaction_manager._pending_partitions_in_transaction -= partitions
+
+        if unauthorized_topics:
+            self.abortable_error(Errors.TopicAuthorizationFailedError(unauthorized_topics))
+        elif has_partition_errors:
+            self.abortable_error(Errors.KafkaError("Could not add partitions to transaction due to errors: %s" % (results)))
+        else:
+            log.debug("Successfully added partitions %s to transaction", partitions)
+            self.transaction_manager._partitions_in_transaction.update(partitions)
+            self.transaction_manager._transaction_started = True
+            self._result.done()
+
+    def maybe_override_retry_backoff_ms(self):
+        # We only want to reduce the backoff when retrying the first AddPartition which errored out due to a
+        # CONCURRENT_TRANSACTIONS error since this means that the previous transaction is still completing and
+        # we don't want to wait too long before trying to start the new one.
+        #
+        # This is only a temporary fix, the long term solution is being tracked in
+        # https://issues.apache.org/jira/browse/KAFKA-5482
+        if not self.transaction_manager._partitions_in_transaction:
+            self.retry_backoff_ms = min(self.transaction_manager.ADD_PARTITIONS_RETRY_BACKOFF_MS, self.retry_backoff_ms)
+
+
+class FindCoordinatorHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, coord_type, coord_key):
+        super(FindCoordinatorHandler, self).__init__(transaction_manager)
+
+        self._coord_type = coord_type
+        self._coord_key = coord_key
+        if transaction_manager._api_version >= (2, 0):
+            version = 2
+        else:
+            version = 1
+        if coord_type == 'group':
+            coord_type_int8 = 0
+        elif coord_type == 'transaction':
+            coord_type_int8 = 1
+        else:
+            raise ValueError("Unrecognized coordinator type: %s" % (coord_type,))
+        self.request = FindCoordinatorRequest[version](
+            coordinator_key=coord_key,
+            coordinator_type=coord_type_int8,
+        )
+
+    @property
+    def priority(self):
+        return Priority.FIND_COORDINATOR
+
+    @property
+    def coordinator_type(self):
+        return None
+
+    @property
+    def coordinator_key(self):
+        return None
+
+    def handle_response(self, response):
+        error = Errors.for_code(response.error_code)
+
+        if error is Errors.NoError:
+            coordinator_id = self.transaction_manager._metadata.add_coordinator(
+                response, self._coord_type, self._coord_key)
+            if self._coord_type == 'group':
+                self.transaction_manager._consumer_group_coordinator = coordinator_id
+            elif self._coord_type == 'transaction':
+                self.transaction_manager._transaction_coordinator = coordinator_id
+            self._result.done()
+        elif error is Errors.CoordinatorNotAvailableError:
+            self.reenqueue()
+        elif error is Errors.TransactionalIdAuthorizationFailedError:
+            self.fatal_error(error())
+        elif error is Errors.GroupAuthorizationFailedError:
+            self.abortable_error(error(self._coord_key))
+        else:
+            self.fatal_error(Errors.KafkaError(
+                "Could not find a coordinator with type %s with key %s due to"
+                " unexpected error: %s" % (self._coord_type, self._coord_key, error())))
+
+
+class EndTxnHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, committed):
+        super(EndTxnHandler, self).__init__(transaction_manager)
+
+        if self.transaction_manager._api_version >= (2, 7):
+            version = 2
+        elif self.transaction_manager._api_version >= (2, 0):
+            version = 1
+        else:
+            version = 0
+        self.request = EndTxnRequest[version](
+            transactional_id=self.transactional_id,
+            producer_id=self.producer_id,
+            producer_epoch=self.producer_epoch,
+            committed=committed)
+
+    @property
+    def priority(self):
+        return Priority.END_TXN
+
+    def handle_response(self, response):
+        error = Errors.for_code(response.error_code)
+
+        if error is Errors.NoError:
+            self.transaction_manager._complete_transaction()
+            self._result.done()
+        elif error in (Errors.CoordinatorNotAvailableError, Errors.NotCoordinatorError):
+            self.transaction_manager._lookup_coordinator('transaction', self.transactional_id)
+            self.reenqueue()
+        elif error in (Errors.CoordinatorLoadInProgressError, Errors.ConcurrentTransactionsError):
+            self.reenqueue()
+        elif error is Errors.InvalidProducerEpochError:
+            self.fatal_error(error())
+        elif error is Errors.TransactionalIdAuthorizationFailedError:
+            self.fatal_error(error())
+        elif error is Errors.InvalidTxnStateError:
+            self.fatal_error(error())
+        else:
+            self.fatal_error(Errors.KafkaError("Unhandled error in EndTxnResponse: %s" % (error())))
+
+
+class AddOffsetsToTxnHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, consumer_group_id, offsets):
+        super(AddOffsetsToTxnHandler, self).__init__(transaction_manager)
+
+        self.consumer_group_id = consumer_group_id
+        self.offsets = offsets
+        if self.transaction_manager._api_version >= (2, 7):
+            version = 2
+        elif self.transaction_manager._api_version >= (2, 0):
+            version = 1
+        else:
+            version = 0
+        self.request = AddOffsetsToTxnRequest[version](
+            transactional_id=self.transactional_id,
+            producer_id=self.producer_id,
+            producer_epoch=self.producer_epoch,
+            group_id=consumer_group_id)
+
+    @property
+    def priority(self):
+        return Priority.ADD_PARTITIONS_OR_OFFSETS
+
+    def handle_response(self, response):
+        error = Errors.for_code(response.error_code)
+
+        if error is Errors.NoError:
+            log.debug("Successfully added partition for consumer group %s to transaction", self.consumer_group_id)
+
+            # note the result is not completed until the TxnOffsetCommit returns
+            for tp, offset in six.iteritems(self.offsets):
+                self.transaction_manager._pending_txn_offset_commits[tp] = offset
+            handler = TxnOffsetCommitHandler(self.transaction_manager, self.consumer_group_id,
+                                             self.transaction_manager._pending_txn_offset_commits, self._result)
+            self.transaction_manager._enqueue_request(handler)
+            self.transaction_manager._transaction_started = True
+        elif error in (Errors.CoordinatorNotAvailableError, Errors.NotCoordinatorError):
+            self.transaction_manager._lookup_coordinator('transaction', self.transactional_id)
+            self.reenqueue()
+        elif error in (Errors.CoordinatorLoadInProgressError, Errors.ConcurrentTransactionsError):
+            self.reenqueue()
+        elif error is Errors.InvalidProducerEpochError:
+            self.fatal_error(error())
+        elif error is Errors.TransactionalIdAuthorizationFailedError:
+            self.fatal_error(error())
+        elif error is Errors.GroupAuthorizationFailedError:
+            self.abortable_error(error(self.consumer_group_id))
+        else:
+            self.fatal_error(Errors.KafkaError("Unexpected error in AddOffsetsToTxnResponse: %s" % (error())))
+
+
+class TxnOffsetCommitHandler(TxnRequestHandler):
+    def __init__(self, transaction_manager, consumer_group_id, offsets, result):
+        super(TxnOffsetCommitHandler, self).__init__(transaction_manager, result=result)
+
+        self.consumer_group_id = consumer_group_id
+        self.offsets = offsets
+        self.request = self._build_request()
+
+    def _build_request(self):
+        if self.transaction_manager._api_version >= (2, 1):
+            version = 2
+        elif self.transaction_manager._api_version >= (2, 0):
+            version = 1
+        else:
+            version = 0
+
+        topic_data = collections.defaultdict(list)
+        for tp, offset in six.iteritems(self.offsets):
+            if version >= 2:
+                partition_data = (tp.partition, offset.offset, offset.leader_epoch, offset.metadata)
+            else:
+                partition_data = (tp.partition, offset.offset, offset.metadata)
+            topic_data[tp.topic].append(partition_data)
+
+        return TxnOffsetCommitRequest[version](
+            transactional_id=self.transactional_id,
+            group_id=self.consumer_group_id,
+            producer_id=self.producer_id,
+            producer_epoch=self.producer_epoch,
+            topics=list(topic_data.items()))
+
+    @property
+    def priority(self):
+        return Priority.ADD_PARTITIONS_OR_OFFSETS
+
+    @property
+    def coordinator_type(self):
+        return 'group'
+
+    @property
+    def coordinator_key(self):
+        return self.consumer_group_id
+
+    def handle_response(self, response):
+        lookup_coordinator = False
+        retriable_failure = False
+
+        errors = {TopicPartition(topic, partition): Errors.for_code(error_code)
+                  for topic, partition_data in response.topics
+                  for partition, error_code in partition_data}
+
+        for tp, error in six.iteritems(errors):
+            if error is Errors.NoError:
+                log.debug("Successfully added offsets for %s from consumer group %s to transaction.",
+                          tp, self.consumer_group_id)
+                del self.transaction_manager._pending_txn_offset_commits[tp]
+            elif error in (errors.CoordinatorNotAvailableError, Errors.NotCoordinatorError, Errors.RequestTimedOutError):
+                retriable_failure = True
+                lookup_coordinator = True
+            elif error is Errors.UnknownTopicOrPartitionError:
+                retriable_failure = True
+            elif error is Errors.GroupAuthorizationFailedError:
+                self.abortable_error(error(self.consumer_group_id))
+                return
+            elif error in (Errors.TransactionalIdAuthorizationFailedError,
+                           Errors.InvalidProducerEpochError,
+                           Errors.UnsupportedForMessageFormatError):
+                self.fatal_error(error())
+                return
+            else:
+                self.fatal_error(Errors.KafkaError("Unexpected error in TxnOffsetCommitResponse: %s" % (error())))
+                return
+
+        if lookup_coordinator:
+            self.transaction_manager._lookup_coordinator('group', self.consumer_group_id)
+
+        if not retriable_failure:
+            # all attempted partitions were either successful, or there was a fatal failure.
+            # either way, we are not retrying, so complete the request.
+            self.result.done()
+
+        # retry the commits which failed with a retriable error.
+        elif self.transaction_manager._pending_txn_offset_commits:
+            self.offsets = self.transaction_manager._pending_txn_offset_commits
+            self.request = self._build_request()
+            self.reenqueue()

venv/lib/python3.11/site-packages/kafka/protocol/__init__.py

@@ -0,0 +1,49 @@
+from __future__ import absolute_import
+
+
+API_KEYS = {
+    0: 'Produce',
+    1: 'Fetch',
+    2: 'ListOffsets',
+    3: 'Metadata',
+    4: 'LeaderAndIsr',
+    5: 'StopReplica',
+    6: 'UpdateMetadata',
+    7: 'ControlledShutdown',
+    8: 'OffsetCommit',
+    9: 'OffsetFetch',
+    10: 'FindCoordinator',
+    11: 'JoinGroup',
+    12: 'Heartbeat',
+    13: 'LeaveGroup',
+    14: 'SyncGroup',
+    15: 'DescribeGroups',
+    16: 'ListGroups',
+    17: 'SaslHandshake',
+    18: 'ApiVersions',
+    19: 'CreateTopics',
+    20: 'DeleteTopics',
+    21: 'DeleteRecords',
+    22: 'InitProducerId',
+    23: 'OffsetForLeaderEpoch',
+    24: 'AddPartitionsToTxn',
+    25: 'AddOffsetsToTxn',
+    26: 'EndTxn',
+    27: 'WriteTxnMarkers',
+    28: 'TxnOffsetCommit',
+    29: 'DescribeAcls',
+    30: 'CreateAcls',
+    31: 'DeleteAcls',
+    32: 'DescribeConfigs',
+    33: 'AlterConfigs',
+    36: 'SaslAuthenticate',
+    37: 'CreatePartitions',
+    38: 'CreateDelegationToken',
+    39: 'RenewDelegationToken',
+    40: 'ExpireDelegationToken',
+    41: 'DescribeDelegationToken',
+    42: 'DeleteGroups',
+    45: 'AlterPartitionReassignments',
+    46: 'ListPartitionReassignments',
+    48: 'DescribeClientQuotas',
+}

venv/lib/python3.11/site-packages/kafka/protocol/abstract.py

@@ -0,0 +1,20 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class AbstractType(object):
+    @abc.abstractmethod
+    def encode(cls, value): # pylint: disable=no-self-argument
+        pass
+
+    @abc.abstractmethod
+    def decode(cls, data): # pylint: disable=no-self-argument
+        pass
+
+    @classmethod
+    def repr(cls, value):
+        return repr(value)

venv/lib/python3.11/site-packages/kafka/protocol/add_offsets_to_txn.py

@@ -0,0 +1,59 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Int16, Int32, Int64, Schema, String
+
+
+class AddOffsetsToTxnResponse_v0(Response):
+    API_KEY = 25
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+    )
+
+
+class AddOffsetsToTxnResponse_v1(Response):
+    API_KEY = 25
+    API_VERSION = 1
+    SCHEMA = AddOffsetsToTxnResponse_v0.SCHEMA
+
+
+class AddOffsetsToTxnResponse_v2(Response):
+    API_KEY = 25
+    API_VERSION = 2
+    SCHEMA = AddOffsetsToTxnResponse_v1.SCHEMA
+
+
+class AddOffsetsToTxnRequest_v0(Request):
+    API_KEY = 25
+    API_VERSION = 0
+    RESPONSE_TYPE = AddOffsetsToTxnResponse_v0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('group_id', String('utf-8')),
+    )
+
+
+class AddOffsetsToTxnRequest_v1(Request):
+    API_KEY = 25
+    API_VERSION = 1
+    RESPONSE_TYPE = AddOffsetsToTxnResponse_v1
+    SCHEMA = AddOffsetsToTxnRequest_v0.SCHEMA
+
+
+class AddOffsetsToTxnRequest_v2(Request):
+    API_KEY = 25
+    API_VERSION = 2
+    RESPONSE_TYPE = AddOffsetsToTxnResponse_v2
+    SCHEMA = AddOffsetsToTxnRequest_v1.SCHEMA
+
+
+AddOffsetsToTxnRequest = [
+    AddOffsetsToTxnRequest_v0, AddOffsetsToTxnRequest_v1, AddOffsetsToTxnRequest_v2,
+]
+AddOffsetsToTxnResponse = [
+    AddOffsetsToTxnResponse_v0, AddOffsetsToTxnResponse_v1, AddOffsetsToTxnResponse_v2,
+]

venv/lib/python3.11/site-packages/kafka/protocol/add_partitions_to_txn.py

@@ -0,0 +1,63 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int16, Int32, Int64, Schema, String
+
+
+class AddPartitionsToTxnResponse_v0(Response):
+    API_KEY = 24
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('results', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16))))))
+
+
+class AddPartitionsToTxnResponse_v1(Response):
+    API_KEY = 24
+    API_VERSION = 1
+    SCHEMA = AddPartitionsToTxnResponse_v0.SCHEMA
+
+
+class AddPartitionsToTxnResponse_v2(Response):
+    API_KEY = 24
+    API_VERSION = 2
+    SCHEMA = AddPartitionsToTxnResponse_v1.SCHEMA
+
+
+class AddPartitionsToTxnRequest_v0(Request):
+    API_KEY = 24
+    API_VERSION = 0
+    RESPONSE_TYPE = AddPartitionsToTxnResponse_v0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32)))))
+
+
+class AddPartitionsToTxnRequest_v1(Request):
+    API_KEY = 24
+    API_VERSION = 1
+    RESPONSE_TYPE = AddPartitionsToTxnResponse_v1
+    SCHEMA = AddPartitionsToTxnRequest_v0.SCHEMA
+
+
+class AddPartitionsToTxnRequest_v2(Request):
+    API_KEY = 24
+    API_VERSION = 2
+    RESPONSE_TYPE = AddPartitionsToTxnResponse_v2
+    SCHEMA = AddPartitionsToTxnRequest_v1.SCHEMA
+
+
+AddPartitionsToTxnRequest = [
+    AddPartitionsToTxnRequest_v0, AddPartitionsToTxnRequest_v1, AddPartitionsToTxnRequest_v2,
+]
+AddPartitionsToTxnResponse = [
+    AddPartitionsToTxnResponse_v0, AddPartitionsToTxnResponse_v1, AddPartitionsToTxnResponse_v2,
+]

venv/lib/python3.11/site-packages/kafka/protocol/api.py

@@ -0,0 +1,141 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.protocol.struct import Struct
+from kafka.protocol.types import Int16, Int32, String, Schema, Array, TaggedFields
+
+from kafka.vendor.six import add_metaclass
+
+
+class RequestHeader(Struct):
+    SCHEMA = Schema(
+        ('api_key', Int16),
+        ('api_version', Int16),
+        ('correlation_id', Int32),
+        ('client_id', String('utf-8'))
+    )
+
+    def __init__(self, request, correlation_id=0, client_id='kafka-python'):
+        super(RequestHeader, self).__init__(
+            request.API_KEY, request.API_VERSION, correlation_id, client_id
+        )
+
+
+class RequestHeaderV2(Struct):
+    # Flexible response / request headers end in field buffer
+    SCHEMA = Schema(
+        ('api_key', Int16),
+        ('api_version', Int16),
+        ('correlation_id', Int32),
+        ('client_id', String('utf-8')),
+        ('tags', TaggedFields),
+    )
+
+    def __init__(self, request, correlation_id=0, client_id='kafka-python', tags=None):
+        super(RequestHeaderV2, self).__init__(
+            request.API_KEY, request.API_VERSION, correlation_id, client_id, tags or {}
+        )
+
+
+class ResponseHeader(Struct):
+    SCHEMA = Schema(
+        ('correlation_id', Int32),
+    )
+
+
+class ResponseHeaderV2(Struct):
+    SCHEMA = Schema(
+        ('correlation_id', Int32),
+        ('tags', TaggedFields),
+    )
+
+
+@add_metaclass(abc.ABCMeta)
+class Request(Struct):
+    FLEXIBLE_VERSION = False
+
+    @abc.abstractproperty
+    def API_KEY(self):
+        """Integer identifier for api request"""
+        pass
+
+    @abc.abstractproperty
+    def API_VERSION(self):
+        """Integer of api request version"""
+        pass
+
+    @abc.abstractproperty
+    def SCHEMA(self):
+        """An instance of Schema() representing the request structure"""
+        pass
+
+    @abc.abstractproperty
+    def RESPONSE_TYPE(self):
+        """The Response class associated with the api request"""
+        pass
+
+    def expect_response(self):
+        """Override this method if an api request does not always generate a response"""
+        return True
+
+    def to_object(self):
+        return _to_object(self.SCHEMA, self)
+
+    def build_header(self, correlation_id, client_id):
+        if self.FLEXIBLE_VERSION:
+            return RequestHeaderV2(self, correlation_id=correlation_id, client_id=client_id)
+        return RequestHeader(self, correlation_id=correlation_id, client_id=client_id)
+
+
+@add_metaclass(abc.ABCMeta)
+class Response(Struct):
+    FLEXIBLE_VERSION = False
+
+    @abc.abstractproperty
+    def API_KEY(self):
+        """Integer identifier for api request/response"""
+        pass
+
+    @abc.abstractproperty
+    def API_VERSION(self):
+        """Integer of api request/response version"""
+        pass
+
+    @abc.abstractproperty
+    def SCHEMA(self):
+        """An instance of Schema() representing the response structure"""
+        pass
+
+    def to_object(self):
+        return _to_object(self.SCHEMA, self)
+
+    @classmethod
+    def parse_header(cls, read_buffer):
+        if cls.FLEXIBLE_VERSION:
+            return ResponseHeaderV2.decode(read_buffer)
+        return ResponseHeader.decode(read_buffer)
+
+
+def _to_object(schema, data):
+    obj = {}
+    for idx, (name, _type) in enumerate(zip(schema.names, schema.fields)):
+        if isinstance(data, Struct):
+            val = data.get_item(name)
+        else:
+            val = data[idx]
+
+        if isinstance(_type, Schema):
+            obj[name] = _to_object(_type, val)
+        elif isinstance(_type, Array):
+            if isinstance(_type.array_of, (Array, Schema)):
+                obj[name] = [
+                    _to_object(_type.array_of, x)
+                    for x in val
+                ]
+            else:
+                obj[name] = val
+        else:
+            obj[name] = val
+
+    return obj

venv/lib/python3.11/site-packages/kafka/protocol/api_versions.py

@@ -0,0 +1,134 @@
+from __future__ import absolute_import
+
+from io import BytesIO
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, CompactArray, CompactString, Int16, Int32, Schema, TaggedFields
+
+
+class BaseApiVersionsResponse(Response):
+    API_KEY = 18
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_versions', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16)))
+    )
+
+    @classmethod
+    def decode(cls, data):
+        if isinstance(data, bytes):
+            data = BytesIO(data)
+        # Check error_code, decode as v0 if any error
+        curr = data.tell()
+        err = Int16.decode(data)
+        data.seek(curr)
+        if err != 0:
+            return ApiVersionsResponse_v0.decode(data)
+        return super(BaseApiVersionsResponse, cls).decode(data)
+
+
+class ApiVersionsResponse_v0(Response):
+    API_KEY = 18
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_versions', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16)))
+    )
+
+
+class ApiVersionsResponse_v1(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_versions', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ApiVersionsResponse_v2(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 2
+    SCHEMA = ApiVersionsResponse_v1.SCHEMA
+
+
+class ApiVersionsResponse_v3(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_versions', CompactArray(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16),
+            ('_tagged_fields', TaggedFields))),
+        ('throttle_time_ms', Int32),
+        ('_tagged_fields', TaggedFields)
+    )
+    # Note: ApiVersions Response does not send FLEXIBLE_VERSION header!
+
+
+class ApiVersionsResponse_v4(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 4
+    SCHEMA = ApiVersionsResponse_v3.SCHEMA
+
+
+class ApiVersionsRequest_v0(Request):
+    API_KEY = 18
+    API_VERSION = 0
+    RESPONSE_TYPE = ApiVersionsResponse_v0
+    SCHEMA = Schema()
+
+
+class ApiVersionsRequest_v1(Request):
+    API_KEY = 18
+    API_VERSION = 1
+    RESPONSE_TYPE = ApiVersionsResponse_v1
+    SCHEMA = ApiVersionsRequest_v0.SCHEMA
+
+
+class ApiVersionsRequest_v2(Request):
+    API_KEY = 18
+    API_VERSION = 2
+    RESPONSE_TYPE = ApiVersionsResponse_v2
+    SCHEMA = ApiVersionsRequest_v1.SCHEMA
+
+
+class ApiVersionsRequest_v3(Request):
+    API_KEY = 18
+    API_VERSION = 3
+    RESPONSE_TYPE = ApiVersionsResponse_v3
+    SCHEMA = Schema(
+        ('client_software_name', CompactString('utf-8')),
+        ('client_software_version', CompactString('utf-8')),
+        ('_tagged_fields', TaggedFields)
+    )
+    FLEXIBLE_VERSION = True
+
+
+class ApiVersionsRequest_v4(Request):
+    API_KEY = 18
+    API_VERSION = 4
+    RESPONSE_TYPE = ApiVersionsResponse_v4
+    SCHEMA = ApiVersionsRequest_v3.SCHEMA
+    FLEXIBLE_VERSION = True
+
+
+ApiVersionsRequest = [
+    ApiVersionsRequest_v0, ApiVersionsRequest_v1, ApiVersionsRequest_v2,
+    ApiVersionsRequest_v3, ApiVersionsRequest_v4,
+]
+ApiVersionsResponse = [
+    ApiVersionsResponse_v0, ApiVersionsResponse_v1, ApiVersionsResponse_v2,
+    ApiVersionsResponse_v3, ApiVersionsResponse_v4,
+]

venv/lib/python3.11/site-packages/kafka/protocol/broker_api_versions.py

@@ -0,0 +1,68 @@
+BROKER_API_VERSIONS = {
+    # api_versions responses prior to (0, 10) are synthesized for compatibility
+    (0, 8, 0): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0)},
+    # adds offset commit + fetch
+    (0, 8, 1): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 8: (0, 0), 9: (0, 0)},
+    # adds find coordinator
+    (0, 8, 2): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 8: (0, 1), 9: (0, 1), 10: (0, 0)},
+    # adds group management (join/sync/leave/heartbeat)
+    (0, 9): {0: (0, 1), 1: (0, 1), 2: (0, 0), 3: (0, 0), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 0), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0)},
+    # adds message format v1, sasl, and api versions api
+    (0, 10, 0): {0: (0, 2), 1: (0, 2), 2: (0, 0), 3: (0, 1), 4: (0, 0), 5: (0, 0), 6: (0, 2), 7: (1, 1), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 0), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0)},
+
+    # All data below is copied from brokers via api_versions_response (see make servers/*/api_versions)
+    # adds admin apis create/delete topics, and bumps fetch/listoffsets/metadata/joingroup
+    (0, 10, 1): {0: (0, 2), 1: (0, 3), 2: (0, 1), 3: (0, 2), 4: (0, 0), 5: (0, 0), 6: (0, 2), 7: (1, 1), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 1), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0), 19: (0, 0), 20: (0, 0)},
+
+    # bumps offsetfetch/create-topics
+    (0, 10, 2): {0: (0, 2), 1: (0, 3), 2: (0, 1), 3: (0, 2), 4: (0, 0), 5: (0, 0), 6: (0, 3), 7: (1, 1), 8: (0, 2), 9: (0, 2), 10: (0, 0), 11: (0, 1), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0), 19: (0, 1), 20: (0, 0)},
+
+    # Adds message format v2, and more admin apis (describe/create/delete acls, describe/alter configs, etc)
+    (0, 11): {0: (0, 3), 1: (0, 5), 2: (0, 2), 3: (0, 4), 4: (0, 0), 5: (0, 0), 6: (0, 3), 7: (1, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 0), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 0), 33: (0, 0)},
+
+    # Adds Sasl Authenticate, and additional admin apis (describe/alter log dirs, etc)
+    (1, 0): {0: (0, 5), 1: (0, 6), 2: (0, 2), 3: (0, 5), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 1), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 0), 33: (0, 0), 34: (0, 0), 35: (0, 0), 36: (0, 0), 37: (0, 0)},
+
+    (1, 1): {0: (0, 5), 1: (0, 7), 2: (0, 2), 3: (0, 5), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 1), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 1), 33: (0, 0), 34: (0, 0), 35: (0, 0), 36: (0, 0), 37: (0, 0), 38: (0, 0), 39: (0, 0), 40: (0, 0), 41: (0, 0), 42: (0, 0)},
+
+    (2, 0): {0: (0, 6), 1: (0, 8), 2: (0, 3), 3: (0, 6), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 4), 9: (0, 4), 10: (0, 2), 11: (0, 3), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 2), 21: (0, 1), 22: (0, 1), 23: (0, 1), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 1), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 0), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1)},
+
+    (2, 1): {0: (0, 7), 1: (0, 10), 2: (0, 4), 3: (0, 7), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 6), 9: (0, 5), 10: (0, 2), 11: (0, 3), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 2), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 0), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1)},
+
+    (2, 2): {0: (0, 7), 1: (0, 10), 2: (0, 5), 3: (0, 7), 4: (0, 2), 5: (0, 1), 6: (0, 5), 7: (0, 2), 8: (0, 6), 9: (0, 5), 10: (0, 2), 11: (0, 4), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 2), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1), 43: (0, 0)},
+
+    (2, 3): {0: (0, 7), 1: (0, 11), 2: (0, 5), 3: (0, 8), 4: (0, 2), 5: (0, 1), 6: (0, 5), 7: (0, 2), 8: (0, 7), 9: (0, 5), 10: (0, 2), 11: (0, 5), 12: (0, 3), 13: (0, 2), 14: (0, 3), 15: (0, 3), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1), 43: (0, 0), 44: (0, 0)},
+
+    (2, 4): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 2), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 6), 10: (0, 3), 11: (0, 6), 12: (0, 4), 13: (0, 4), 14: (0, 4), 15: (0, 5), 16: (0, 3), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 1), 22: (0, 2), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 2), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0)},
+
+    (2, 5): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 2), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 3), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 1), 22: (0, 3), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 2), 37: (0, 2), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0)},
+
+    (2, 6): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 3), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 2), 22: (0, 3), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 3), 33: (0, 1), 34: (0, 1), 35: (0, 2), 36: (0, 2), 37: (0, 2), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 0), 49: (0, 0)},
+
+    (2, 7): {0: (0, 8), 1: (0, 12), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 3), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 6), 20: (0, 5), 21: (0, 2), 22: (0, 4), 23: (0, 3), 24: (0, 2), 25: (0, 2), 26: (0, 2), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 3), 33: (0, 1), 34: (0, 1), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 0), 49: (0, 0), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0)},
+
+    (2, 8): {0: (0, 9), 1: (0, 12), 2: (0, 6), 3: (0, 11), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0)},
+
+    (3, 0): {0: (0, 9), 1: (0, 12), 2: (0, 7), 3: (0, 11), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 1): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 2): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 6), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 3), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 1), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 3): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 6), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 2), 57: (0, 1), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 4): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 2), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 5): {0: (0, 9), 1: (0, 15), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 6): {0: (0, 9), 1: (0, 15), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 4), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)},
+
+    (3, 7): {0: (0, 10), 1: (0, 16), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 4), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0), 68: (0, 0)},
+
+    (3, 8): {0: (0, 11), 1: (0, 16), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 5), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 5), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 5), 23: (0, 4), 24: (0, 5), 25: (0, 4), 26: (0, 4), 27: (0, 1), 28: (0, 4), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 1), 67: (0, 0), 68: (0, 0), 69: (0, 0)},
+
+    (3, 9): {0: (0, 11), 1: (0, 17), 2: (0, 9), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 5), 23: (0, 4), 24: (0, 5), 25: (0, 4), 26: (0, 4), 27: (0, 1), 28: (0, 4), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 1), 67: (0, 0), 68: (0, 0), 69: (0, 0)},
+
+    (4, 0): {0: (0, 12), 1: (4, 17), 2: (1, 10), 3: (0, 13), 8: (2, 9), 9: (1, 9), 10: (0, 6), 11: (2, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 6), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (2, 7), 20: (1, 6), 21: (0, 2), 22: (0, 5), 23: (2, 4), 24: (0, 5), 25: (0, 4), 26: (0, 5), 27: (1, 1), 28: (0, 5), 29: (1, 3), 30: (1, 3), 31: (1, 3), 32: (1, 4), 33: (0, 2), 34: (1, 2), 35: (1, 4), 36: (0, 2), 37: (0, 3), 38: (1, 3), 39: (1, 2), 40: (1, 2), 41: (1, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 55: (0, 2), 57: (0, 2), 60: (0, 2), 61: (0, 0), 64: (0, 0), 65: (0, 0), 66: (0, 1), 68: (0, 1), 69: (0, 1), 74: (0, 0), 75: (0, 0), 80: (0, 0), 81: (0, 0)},
+
+}

venv/lib/python3.11/site-packages/kafka/protocol/commit.py

@@ -0,0 +1,313 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int16, Int32, Int64, Schema, String
+
+
+class OffsetCommitResponse_v0(Response):
+    API_KEY = 8
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetCommitResponse_v1(Response):
+    API_KEY = 8
+    API_VERSION = 1
+    SCHEMA = OffsetCommitResponse_v0.SCHEMA
+
+
+class OffsetCommitResponse_v2(Response):
+    API_KEY = 8
+    API_VERSION = 2
+    SCHEMA = OffsetCommitResponse_v1.SCHEMA
+
+
+class OffsetCommitResponse_v3(Response):
+    API_KEY = 8
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetCommitResponse_v4(Response):
+    API_KEY = 8
+    API_VERSION = 4
+    SCHEMA = OffsetCommitResponse_v3.SCHEMA
+
+
+class OffsetCommitResponse_v5(Response):
+    API_KEY = 8
+    API_VERSION = 5
+    SCHEMA = OffsetCommitResponse_v4.SCHEMA
+
+
+class OffsetCommitResponse_v6(Response):
+    API_KEY = 8
+    API_VERSION = 6
+    SCHEMA = OffsetCommitResponse_v5.SCHEMA
+
+
+class OffsetCommitRequest_v0(Request):
+    API_KEY = 8
+    API_VERSION = 0  # Zookeeper-backed storage
+    RESPONSE_TYPE = OffsetCommitResponse_v0
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8'))))))
+    )
+
+
+class OffsetCommitRequest_v1(Request):
+    API_KEY = 8
+    API_VERSION = 1  # Kafka-backed storage
+    RESPONSE_TYPE = OffsetCommitResponse_v1
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('consumer_group_generation_id', Int32),
+        ('consumer_id', String('utf-8')),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('timestamp', Int64),
+                ('metadata', String('utf-8'))))))
+    )
+
+
+class OffsetCommitRequest_v2(Request):
+    API_KEY = 8
+    API_VERSION = 2
+    RESPONSE_TYPE = OffsetCommitResponse_v2
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('consumer_group_generation_id', Int32),
+        ('consumer_id', String('utf-8')),
+        ('retention_time', Int64), # added retention_time, dropped timestamp
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8'))))))
+    )
+    DEFAULT_RETENTION_TIME = -1
+
+
+class OffsetCommitRequest_v3(Request):
+    API_KEY = 8
+    API_VERSION = 3
+    RESPONSE_TYPE = OffsetCommitResponse_v3
+    SCHEMA = OffsetCommitRequest_v2.SCHEMA
+    DEFAULT_RETENTION_TIME = -1
+
+
+class OffsetCommitRequest_v4(Request):
+    API_KEY = 8
+    API_VERSION = 4
+    RESPONSE_TYPE = OffsetCommitResponse_v4
+    SCHEMA = OffsetCommitRequest_v3.SCHEMA
+    DEFAULT_RETENTION_TIME = -1
+
+
+class OffsetCommitRequest_v5(Request):
+    API_KEY = 8
+    API_VERSION = 5 # drops retention_time
+    RESPONSE_TYPE = OffsetCommitResponse_v5
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('consumer_group_generation_id', Int32),
+        ('consumer_id', String('utf-8')),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8'))))))
+    )
+
+
+class OffsetCommitRequest_v6(Request):
+    API_KEY = 8
+    API_VERSION = 6
+    RESPONSE_TYPE = OffsetCommitResponse_v6
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('consumer_group_generation_id', Int32),
+        ('consumer_id', String('utf-8')),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('leader_epoch', Int32), # added for fencing / kip-320. default -1
+                ('metadata', String('utf-8'))))))
+    )
+
+
+OffsetCommitRequest = [
+    OffsetCommitRequest_v0, OffsetCommitRequest_v1,
+    OffsetCommitRequest_v2, OffsetCommitRequest_v3,
+    OffsetCommitRequest_v4, OffsetCommitRequest_v5,
+    OffsetCommitRequest_v6,
+]
+OffsetCommitResponse = [
+    OffsetCommitResponse_v0, OffsetCommitResponse_v1,
+    OffsetCommitResponse_v2, OffsetCommitResponse_v3,
+    OffsetCommitResponse_v4, OffsetCommitResponse_v5,
+    OffsetCommitResponse_v6,
+]
+
+
+class OffsetFetchResponse_v0(Response):
+    API_KEY = 9
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetFetchResponse_v1(Response):
+    API_KEY = 9
+    API_VERSION = 1
+    SCHEMA = OffsetFetchResponse_v0.SCHEMA
+
+
+class OffsetFetchResponse_v2(Response):
+    # Added in KIP-88
+    API_KEY = 9
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchResponse_v3(Response):
+    API_KEY = 9
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchResponse_v4(Response):
+    API_KEY = 9
+    API_VERSION = 4
+    SCHEMA = OffsetFetchResponse_v3.SCHEMA
+
+
+class OffsetFetchResponse_v5(Response):
+    API_KEY = 9
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('leader_epoch', Int32),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchRequest_v0(Request):
+    API_KEY = 9
+    API_VERSION = 0  # zookeeper-backed storage
+    RESPONSE_TYPE = OffsetFetchResponse_v0
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8')),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32))))
+    )
+
+
+class OffsetFetchRequest_v1(Request):
+    API_KEY = 9
+    API_VERSION = 1  # kafka-backed storage
+    RESPONSE_TYPE = OffsetFetchResponse_v1
+    SCHEMA = OffsetFetchRequest_v0.SCHEMA
+
+
+class OffsetFetchRequest_v2(Request):
+    # KIP-88: Allows passing null topics to return offsets for all partitions
+    # that the consumer group has a stored offset for, even if no consumer in
+    # the group is currently consuming that partition.
+    API_KEY = 9
+    API_VERSION = 2
+    RESPONSE_TYPE = OffsetFetchResponse_v2
+    SCHEMA = OffsetFetchRequest_v1.SCHEMA
+
+
+class OffsetFetchRequest_v3(Request):
+    API_KEY = 9
+    API_VERSION = 3
+    RESPONSE_TYPE = OffsetFetchResponse_v3
+    SCHEMA = OffsetFetchRequest_v2.SCHEMA
+
+
+class OffsetFetchRequest_v4(Request):
+    API_KEY = 9
+    API_VERSION = 4
+    RESPONSE_TYPE = OffsetFetchResponse_v4
+    SCHEMA = OffsetFetchRequest_v3.SCHEMA
+
+
+class OffsetFetchRequest_v5(Request):
+    API_KEY = 9
+    API_VERSION = 5
+    RESPONSE_TYPE = OffsetFetchResponse_v5
+    SCHEMA = OffsetFetchRequest_v4.SCHEMA
+
+
+OffsetFetchRequest = [
+    OffsetFetchRequest_v0, OffsetFetchRequest_v1,
+    OffsetFetchRequest_v2, OffsetFetchRequest_v3,
+    OffsetFetchRequest_v4, OffsetFetchRequest_v5,
+]
+OffsetFetchResponse = [
+    OffsetFetchResponse_v0, OffsetFetchResponse_v1,
+    OffsetFetchResponse_v2, OffsetFetchResponse_v3,
+    OffsetFetchResponse_v4, OffsetFetchResponse_v5,
+]

venv/lib/python3.11/site-packages/kafka/protocol/end_txn.py

@@ -0,0 +1,58 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Boolean, Int16, Int32, Int64, Schema, String
+
+
+class EndTxnResponse_v0(Response):
+    API_KEY = 26
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+    )
+
+
+class EndTxnResponse_v1(Response):
+    API_KEY = 26
+    API_VERSION = 1
+    SCHEMA = EndTxnResponse_v0.SCHEMA
+
+
+class EndTxnResponse_v2(Response):
+    API_KEY = 26
+    API_VERSION = 2
+    SCHEMA = EndTxnResponse_v1.SCHEMA
+
+
+class EndTxnRequest_v0(Request):
+    API_KEY = 26
+    API_VERSION = 0
+    RESPONSE_TYPE = EndTxnResponse_v0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('committed', Boolean))
+
+
+class EndTxnRequest_v1(Request):
+    API_KEY = 26
+    API_VERSION = 1
+    RESPONSE_TYPE = EndTxnResponse_v1
+    SCHEMA = EndTxnRequest_v0.SCHEMA
+
+
+class EndTxnRequest_v2(Request):
+    API_KEY = 26
+    API_VERSION = 2
+    RESPONSE_TYPE = EndTxnResponse_v2
+    SCHEMA = EndTxnRequest_v1.SCHEMA
+
+
+EndTxnRequest = [
+    EndTxnRequest_v0, EndTxnRequest_v1, EndTxnRequest_v2,
+]
+EndTxnResponse = [
+    EndTxnResponse_v0, EndTxnResponse_v1, EndTxnResponse_v2,
+]

venv/lib/python3.11/site-packages/kafka/protocol/fetch.py

@@ -0,0 +1,394 @@
+from __future__ import absolute_import
+
+import collections
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int8, Int16, Int32, Int64, Schema, String, Bytes
+
+
+AbortedTransaction = collections.namedtuple("AbortedTransaction",
+    ["producer_id", "first_offset"])
+
+
+class FetchResponse_v0(Response):
+    API_KEY = 1
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('records', Bytes)))))
+    )
+
+
+class FetchResponse_v1(Response):
+    API_KEY = 1
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('records', Bytes)))))
+    )
+
+
+class FetchResponse_v2(Response):
+    API_KEY = 1
+    API_VERSION = 2
+    SCHEMA = FetchResponse_v1.SCHEMA  # message format changed internally
+
+
+class FetchResponse_v3(Response):
+    API_KEY = 1
+    API_VERSION = 3
+    SCHEMA = FetchResponse_v2.SCHEMA
+
+
+class FetchResponse_v4(Response):
+    # Adds message format v2
+    API_KEY = 1
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('last_stable_offset', Int64),
+                ('aborted_transactions', Array(
+                    ('producer_id', Int64),
+                    ('first_offset', Int64))),
+                ('records', Bytes)))))
+    )
+
+
+class FetchResponse_v5(Response):
+    API_KEY = 1
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('last_stable_offset', Int64),
+                ('log_start_offset', Int64),
+                ('aborted_transactions', Array(
+                    ('producer_id', Int64),
+                    ('first_offset', Int64))),
+                ('records', Bytes)))))
+    )
+
+
+class FetchResponse_v6(Response):
+    """
+    Same as FetchResponse_v5. The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 5
+    """
+    API_KEY = 1
+    API_VERSION = 6
+    SCHEMA = FetchResponse_v5.SCHEMA
+
+
+class FetchResponse_v7(Response):
+    """
+    Add error_code and session_id to response
+    """
+    API_KEY = 1
+    API_VERSION = 7
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('session_id', Int32),
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('last_stable_offset', Int64),
+                ('log_start_offset', Int64),
+                ('aborted_transactions', Array(
+                    ('producer_id', Int64),
+                    ('first_offset', Int64))),
+                ('records', Bytes)))))
+    )
+
+
+class FetchResponse_v8(Response):
+    API_KEY = 1
+    API_VERSION = 8
+    SCHEMA = FetchResponse_v7.SCHEMA
+
+
+class FetchResponse_v9(Response):
+    API_KEY = 1
+    API_VERSION = 9
+    SCHEMA = FetchResponse_v7.SCHEMA
+
+
+class FetchResponse_v10(Response):
+    API_KEY = 1
+    API_VERSION = 10
+    SCHEMA = FetchResponse_v7.SCHEMA
+
+
+class FetchResponse_v11(Response):
+    API_KEY = 1
+    API_VERSION = 11
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('session_id', Int32),
+        ('topics', Array(
+            ('topics', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('highwater_offset', Int64),
+                ('last_stable_offset', Int64),
+                ('log_start_offset', Int64),
+                ('aborted_transactions', Array(
+                    ('producer_id', Int64),
+                    ('first_offset', Int64))),
+                ('preferred_read_replica', Int32),
+                ('records', Bytes)))))
+    )
+
+
+class FetchRequest_v0(Request):
+    API_KEY = 1
+    API_VERSION = 0
+    RESPONSE_TYPE = FetchResponse_v0
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('max_bytes', Int32)))))
+    )
+
+
+class FetchRequest_v1(Request):
+    API_KEY = 1
+    API_VERSION = 1
+    RESPONSE_TYPE = FetchResponse_v1
+    SCHEMA = FetchRequest_v0.SCHEMA
+
+
+class FetchRequest_v2(Request):
+    API_KEY = 1
+    API_VERSION = 2
+    RESPONSE_TYPE = FetchResponse_v2
+    SCHEMA = FetchRequest_v1.SCHEMA
+
+
+class FetchRequest_v3(Request):
+    API_KEY = 1
+    API_VERSION = 3
+    RESPONSE_TYPE = FetchResponse_v3
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),  # This new field is only difference from FR_v2
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('max_bytes', Int32)))))
+    )
+
+
+class FetchRequest_v4(Request):
+    # Adds isolation_level field
+    # Adds message format v2
+    API_KEY = 1
+    API_VERSION = 4
+    RESPONSE_TYPE = FetchResponse_v4
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),
+        ('isolation_level', Int8),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('max_bytes', Int32)))))
+    )
+
+
+class FetchRequest_v5(Request):
+    # This may only be used in broker-broker api calls
+    API_KEY = 1
+    API_VERSION = 5
+    RESPONSE_TYPE = FetchResponse_v5
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),
+        ('isolation_level', Int8),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('fetch_offset', Int64),
+                ('log_start_offset', Int64),
+                ('max_bytes', Int32)))))
+    )
+
+
+class FetchRequest_v6(Request):
+    """
+    The body of FETCH_REQUEST_V6 is the same as FETCH_REQUEST_V5.
+    The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 5
+    """
+    API_KEY = 1
+    API_VERSION = 6
+    RESPONSE_TYPE = FetchResponse_v6
+    SCHEMA = FetchRequest_v5.SCHEMA
+
+
+class FetchRequest_v7(Request):
+    """
+    Add incremental fetch requests (see KIP-227)
+    """
+    API_KEY = 1
+    API_VERSION = 7
+    RESPONSE_TYPE = FetchResponse_v7
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),
+        ('isolation_level', Int8),
+        ('session_id', Int32),
+        ('session_epoch', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('fetch_offset', Int64),
+                ('log_start_offset', Int64),
+                ('max_bytes', Int32))))),
+        ('forgotten_topics_data', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32))
+        )),
+    )
+
+
+class FetchRequest_v8(Request):
+    """
+    bump used to indicate that on quota violation brokers send out responses before throttling.
+    """
+    API_KEY = 1
+    API_VERSION = 8
+    RESPONSE_TYPE = FetchResponse_v8
+    SCHEMA = FetchRequest_v7.SCHEMA
+
+
+class FetchRequest_v9(Request):
+    """
+    adds the current leader epoch (see KIP-320)
+    """
+    API_KEY = 1
+    API_VERSION = 9
+    RESPONSE_TYPE = FetchResponse_v9
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),
+        ('isolation_level', Int8),
+        ('session_id', Int32),
+        ('session_epoch', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('fetch_offset', Int64),
+                ('log_start_offset', Int64),
+                ('max_bytes', Int32))))),
+        ('forgotten_topics_data', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32)),
+        )),
+    )
+
+
+class FetchRequest_v10(Request):
+    """
+    bumped up to indicate ZStandard capability. (see KIP-110)
+    """
+    API_KEY = 1
+    API_VERSION = 10
+    RESPONSE_TYPE = FetchResponse_v10
+    SCHEMA = FetchRequest_v9.SCHEMA
+
+
+class FetchRequest_v11(Request):
+    """
+    added rack ID to support read from followers (KIP-392)
+    """
+    API_KEY = 1
+    API_VERSION = 11
+    RESPONSE_TYPE = FetchResponse_v11
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('max_wait_time', Int32),
+        ('min_bytes', Int32),
+        ('max_bytes', Int32),
+        ('isolation_level', Int8),
+        ('session_id', Int32),
+        ('session_epoch', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('fetch_offset', Int64),
+                ('log_start_offset', Int64),
+                ('max_bytes', Int32))))),
+        ('forgotten_topics_data', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32))
+        )),
+        ('rack_id', String('utf-8')),
+    )
+
+
+FetchRequest = [
+    FetchRequest_v0, FetchRequest_v1, FetchRequest_v2,
+    FetchRequest_v3, FetchRequest_v4, FetchRequest_v5,
+    FetchRequest_v6, FetchRequest_v7, FetchRequest_v8,
+    FetchRequest_v9, FetchRequest_v10, FetchRequest_v11,
+]
+FetchResponse = [
+    FetchResponse_v0, FetchResponse_v1, FetchResponse_v2,
+    FetchResponse_v3, FetchResponse_v4, FetchResponse_v5,
+    FetchResponse_v6, FetchResponse_v7, FetchResponse_v8,
+    FetchResponse_v9, FetchResponse_v10, FetchResponse_v11,
+]

venv/lib/python3.11/site-packages/kafka/protocol/find_coordinator.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Int8, Int16, Int32, Schema, String
+
+
+class FindCoordinatorResponse_v0(Response):
+    API_KEY = 10
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('coordinator_id', Int32),
+        ('host', String('utf-8')),
+        ('port', Int32)
+    )
+
+
+class FindCoordinatorResponse_v1(Response):
+    API_KEY = 10
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('error_message', String('utf-8')),
+        ('coordinator_id', Int32),
+        ('host', String('utf-8')),
+        ('port', Int32)
+    )
+
+
+class FindCoordinatorResponse_v2(Response):
+    API_KEY = 10
+    API_VERSION = 2
+    SCHEMA = FindCoordinatorResponse_v1.SCHEMA
+
+
+class FindCoordinatorRequest_v0(Request):
+    API_KEY = 10
+    API_VERSION = 0
+    RESPONSE_TYPE = FindCoordinatorResponse_v0
+    SCHEMA = Schema(
+        ('consumer_group', String('utf-8'))
+    )
+
+
+class FindCoordinatorRequest_v1(Request):
+    API_KEY = 10
+    API_VERSION = 1
+    RESPONSE_TYPE = FindCoordinatorResponse_v1
+    SCHEMA = Schema(
+        ('coordinator_key', String('utf-8')),
+        ('coordinator_type', Int8) # 0: consumer, 1: transaction
+    )
+
+
+class FindCoordinatorRequest_v2(Request):
+    API_KEY = 10
+    API_VERSION = 2
+    RESPONSE_TYPE = FindCoordinatorResponse_v2
+    SCHEMA = FindCoordinatorRequest_v1.SCHEMA
+
+
+FindCoordinatorRequest = [FindCoordinatorRequest_v0, FindCoordinatorRequest_v1, FindCoordinatorRequest_v2]
+FindCoordinatorResponse = [FindCoordinatorResponse_v0, FindCoordinatorResponse_v1, FindCoordinatorResponse_v2]

venv/lib/python3.11/site-packages/kafka/protocol/frame.py

@@ -0,0 +1,30 @@
+class KafkaBytes(bytearray):
+    def __init__(self, size):
+        super(KafkaBytes, self).__init__(size)
+        self._idx = 0
+
+    def read(self, nbytes=None):
+        if nbytes is None:
+            nbytes = len(self) - self._idx
+        start = self._idx
+        self._idx += nbytes
+        if self._idx > len(self):
+            self._idx = len(self)
+        return bytes(self[start:self._idx])
+
+    def write(self, data):
+        start = self._idx
+        self._idx += len(data)
+        self[start:self._idx] = data
+
+    def seek(self, idx):
+        self._idx = idx
+
+    def tell(self):
+        return self._idx
+
+    def __str__(self):
+        return 'KafkaBytes(%d)' % len(self)
+
+    def __repr__(self):
+        return str(self)

venv/lib/python3.11/site-packages/kafka/protocol/group.py

@@ -0,0 +1,298 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.struct import Struct
+from kafka.protocol.types import Array, Bytes, Int16, Int32, Schema, String
+
+
+DEFAULT_GENERATION_ID = -1
+UNKNOWN_MEMBER_ID = ''
+
+
+class JoinGroupResponse_v0(Response):
+    API_KEY = 11
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('generation_id', Int32),
+        ('group_protocol', String('utf-8')),
+        ('leader_id', String('utf-8')),
+        ('member_id', String('utf-8')),
+        ('members', Array(
+            ('member_id', String('utf-8')),
+            ('member_metadata', Bytes)))
+    )
+
+
+class JoinGroupResponse_v1(Response):
+    API_KEY = 11
+    API_VERSION = 1
+    SCHEMA = JoinGroupResponse_v0.SCHEMA
+
+
+class JoinGroupResponse_v2(Response):
+    API_KEY = 11
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('generation_id', Int32),
+        ('group_protocol', String('utf-8')),
+        ('leader_id', String('utf-8')),
+        ('member_id', String('utf-8')),
+        ('members', Array(
+            ('member_id', String('utf-8')),
+            ('member_metadata', Bytes)))
+    )
+
+
+class JoinGroupResponse_v3(Response):
+    API_KEY = 11
+    API_VERSION = 3
+    SCHEMA = JoinGroupResponse_v2.SCHEMA
+
+
+class JoinGroupResponse_v4(Response):
+    API_KEY = 11
+    API_VERSION = 4
+    SCHEMA = JoinGroupResponse_v3.SCHEMA
+
+
+class JoinGroupRequest_v0(Request):
+    API_KEY = 11
+    API_VERSION = 0
+    RESPONSE_TYPE = JoinGroupResponse_v0
+    SCHEMA = Schema(
+        ('group', String('utf-8')),
+        ('session_timeout', Int32),
+        ('member_id', String('utf-8')),
+        ('protocol_type', String('utf-8')),
+        ('group_protocols', Array(
+            ('protocol_name', String('utf-8')),
+            ('protocol_metadata', Bytes)))
+    )
+
+
+class JoinGroupRequest_v1(Request):
+    API_KEY = 11
+    API_VERSION = 1
+    RESPONSE_TYPE = JoinGroupResponse_v1
+    SCHEMA = Schema(
+        ('group', String('utf-8')),
+        ('session_timeout', Int32),
+        ('rebalance_timeout', Int32),
+        ('member_id', String('utf-8')),
+        ('protocol_type', String('utf-8')),
+        ('group_protocols', Array(
+            ('protocol_name', String('utf-8')),
+            ('protocol_metadata', Bytes)))
+    )
+
+
+class JoinGroupRequest_v2(Request):
+    API_KEY = 11
+    API_VERSION = 2
+    RESPONSE_TYPE = JoinGroupResponse_v2
+    SCHEMA = JoinGroupRequest_v1.SCHEMA
+
+
+class JoinGroupRequest_v3(Request):
+    API_KEY = 11
+    API_VERSION = 3
+    RESPONSE_TYPE = JoinGroupResponse_v3
+    SCHEMA = JoinGroupRequest_v2.SCHEMA
+
+
+class JoinGroupRequest_v4(Request):
+    API_KEY = 11
+    API_VERSION = 4
+    RESPONSE_TYPE = JoinGroupResponse_v4
+    SCHEMA = JoinGroupRequest_v3.SCHEMA
+
+
+JoinGroupRequest = [
+    JoinGroupRequest_v0, JoinGroupRequest_v1, JoinGroupRequest_v2,
+    JoinGroupRequest_v3, JoinGroupRequest_v4,
+]
+JoinGroupResponse = [
+    JoinGroupResponse_v0, JoinGroupResponse_v1, JoinGroupResponse_v2,
+    JoinGroupResponse_v3, JoinGroupResponse_v4,
+]
+
+
+class ProtocolMetadata(Struct):
+    SCHEMA = Schema(
+        ('version', Int16),
+        ('subscription', Array(String('utf-8'))), # topics list
+        ('user_data', Bytes)
+    )
+
+
+class SyncGroupResponse_v0(Response):
+    API_KEY = 14
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('member_assignment', Bytes)
+    )
+
+
+class SyncGroupResponse_v1(Response):
+    API_KEY = 14
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('member_assignment', Bytes)
+    )
+
+
+class SyncGroupResponse_v2(Response):
+    API_KEY = 14
+    API_VERSION = 2
+    SCHEMA = SyncGroupResponse_v1.SCHEMA
+
+
+class SyncGroupRequest_v0(Request):
+    API_KEY = 14
+    API_VERSION = 0
+    RESPONSE_TYPE = SyncGroupResponse_v0
+    SCHEMA = Schema(
+        ('group', String('utf-8')),
+        ('generation_id', Int32),
+        ('member_id', String('utf-8')),
+        ('group_assignment', Array(
+            ('member_id', String('utf-8')),
+            ('member_metadata', Bytes)))
+    )
+
+
+class SyncGroupRequest_v1(Request):
+    API_KEY = 14
+    API_VERSION = 1
+    RESPONSE_TYPE = SyncGroupResponse_v1
+    SCHEMA = SyncGroupRequest_v0.SCHEMA
+
+
+class SyncGroupRequest_v2(Request):
+    API_KEY = 14
+    API_VERSION = 2
+    RESPONSE_TYPE = SyncGroupResponse_v2
+    SCHEMA = SyncGroupRequest_v1.SCHEMA
+
+
+SyncGroupRequest = [SyncGroupRequest_v0, SyncGroupRequest_v1, SyncGroupRequest_v2]
+SyncGroupResponse = [SyncGroupResponse_v0, SyncGroupResponse_v1, SyncGroupResponse_v2]
+
+
+class MemberAssignment(Struct):
+    SCHEMA = Schema(
+        ('version', Int16),
+        ('assignment', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(Int32)))),
+        ('user_data', Bytes)
+    )
+
+
+class HeartbeatResponse_v0(Response):
+    API_KEY = 12
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16)
+    )
+
+
+class HeartbeatResponse_v1(Response):
+    API_KEY = 12
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16)
+    )
+
+
+class HeartbeatResponse_v2(Response):
+    API_KEY = 12
+    API_VERSION = 2
+    SCHEMA = HeartbeatResponse_v1.SCHEMA
+
+
+class HeartbeatRequest_v0(Request):
+    API_KEY = 12
+    API_VERSION = 0
+    RESPONSE_TYPE = HeartbeatResponse_v0
+    SCHEMA = Schema(
+        ('group', String('utf-8')),
+        ('generation_id', Int32),
+        ('member_id', String('utf-8'))
+    )
+
+
+class HeartbeatRequest_v1(Request):
+    API_KEY = 12
+    API_VERSION = 1
+    RESPONSE_TYPE = HeartbeatResponse_v1
+    SCHEMA = HeartbeatRequest_v0.SCHEMA
+
+
+class HeartbeatRequest_v2(Request):
+    API_KEY = 12
+    API_VERSION = 2
+    RESPONSE_TYPE = HeartbeatResponse_v2
+    SCHEMA = HeartbeatRequest_v1.SCHEMA
+
+
+HeartbeatRequest = [HeartbeatRequest_v0, HeartbeatRequest_v1, HeartbeatRequest_v2]
+HeartbeatResponse = [HeartbeatResponse_v0, HeartbeatResponse_v1, HeartbeatResponse_v2]
+
+
+class LeaveGroupResponse_v0(Response):
+    API_KEY = 13
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16)
+    )
+
+
+class LeaveGroupResponse_v1(Response):
+    API_KEY = 13
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16)
+    )
+
+
+class LeaveGroupResponse_v2(Response):
+    API_KEY = 13
+    API_VERSION = 2
+    SCHEMA = LeaveGroupResponse_v1.SCHEMA
+
+
+class LeaveGroupRequest_v0(Request):
+    API_KEY = 13
+    API_VERSION = 0
+    RESPONSE_TYPE = LeaveGroupResponse_v0
+    SCHEMA = Schema(
+        ('group', String('utf-8')),
+        ('member_id', String('utf-8'))
+    )
+
+
+class LeaveGroupRequest_v1(Request):
+    API_KEY = 13
+    API_VERSION = 1
+    RESPONSE_TYPE = LeaveGroupResponse_v1
+    SCHEMA = LeaveGroupRequest_v0.SCHEMA
+
+
+class LeaveGroupRequest_v2(Request):
+    API_KEY = 13
+    API_VERSION = 2
+    RESPONSE_TYPE = LeaveGroupResponse_v2
+    SCHEMA = LeaveGroupRequest_v1.SCHEMA
+
+
+LeaveGroupRequest = [LeaveGroupRequest_v0, LeaveGroupRequest_v1, LeaveGroupRequest_v2]
+LeaveGroupResponse = [LeaveGroupResponse_v0, LeaveGroupResponse_v1, LeaveGroupResponse_v2]

venv/lib/python3.11/site-packages/kafka/protocol/init_producer_id.py

@@ -0,0 +1,46 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Int16, Int32, Int64, Schema, String
+
+
+class InitProducerIdResponse_v0(Response):
+    API_KEY = 22
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('error_code', Int16),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+    )
+
+
+class InitProducerIdResponse_v1(Response):
+    API_KEY = 22
+    API_VERSION = 1
+    SCHEMA = InitProducerIdResponse_v0.SCHEMA
+
+
+class InitProducerIdRequest_v0(Request):
+    API_KEY = 22
+    API_VERSION = 0
+    RESPONSE_TYPE = InitProducerIdResponse_v0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('transaction_timeout_ms', Int32),
+    )
+
+
+class InitProducerIdRequest_v1(Request):
+    API_KEY = 22
+    API_VERSION = 1
+    RESPONSE_TYPE = InitProducerIdResponse_v1
+    SCHEMA = InitProducerIdRequest_v0.SCHEMA
+
+
+InitProducerIdRequest = [
+    InitProducerIdRequest_v0, InitProducerIdRequest_v1,
+]
+InitProducerIdResponse = [
+    InitProducerIdResponse_v0, InitProducerIdResponse_v1,
+]

venv/lib/python3.11/site-packages/kafka/protocol/list_offsets.py

@@ -0,0 +1,194 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int8, Int16, Int32, Int64, Schema, String
+
+UNKNOWN_OFFSET = -1
+
+
+class OffsetResetStrategy(object):
+    LATEST = -1
+    EARLIEST = -2
+    NONE = 0
+
+
+class ListOffsetsResponse_v0(Response):
+    API_KEY = 2
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offsets', Array(Int64))))))
+    )
+
+class ListOffsetsResponse_v1(Response):
+    API_KEY = 2
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('timestamp', Int64),
+                ('offset', Int64)))))
+    )
+
+
+class ListOffsetsResponse_v2(Response):
+    API_KEY = 2
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('timestamp', Int64),
+                ('offset', Int64)))))
+    )
+
+
+class ListOffsetsResponse_v3(Response):
+    """
+    on quota violation, brokers send out responses before throttling
+    """
+    API_KEY = 2
+    API_VERSION = 3
+    SCHEMA = ListOffsetsResponse_v2.SCHEMA
+
+
+class ListOffsetsResponse_v4(Response):
+    """
+    Add leader_epoch to response
+    """
+    API_KEY = 2
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('timestamp', Int64),
+                ('offset', Int64),
+                ('leader_epoch', Int32)))))
+    )
+
+
+class ListOffsetsResponse_v5(Response):
+    """
+    adds a new error code, OFFSET_NOT_AVAILABLE
+    """
+    API_KEY = 2
+    API_VERSION = 5
+    SCHEMA = ListOffsetsResponse_v4.SCHEMA
+
+
+class ListOffsetsRequest_v0(Request):
+    API_KEY = 2
+    API_VERSION = 0
+    RESPONSE_TYPE = ListOffsetsResponse_v0
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('timestamp', Int64),
+                ('max_offsets', Int32)))))
+    )
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+class ListOffsetsRequest_v1(Request):
+    API_KEY = 2
+    API_VERSION = 1
+    RESPONSE_TYPE = ListOffsetsResponse_v1
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('timestamp', Int64)))))
+    )
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+
+class ListOffsetsRequest_v2(Request):
+    API_KEY = 2
+    API_VERSION = 2
+    RESPONSE_TYPE = ListOffsetsResponse_v2
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('isolation_level', Int8),  # <- added isolation_level
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('timestamp', Int64)))))
+    )
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+
+class ListOffsetsRequest_v3(Request):
+    API_KEY = 2
+    API_VERSION = 3
+    RESPONSE_TYPE = ListOffsetsResponse_v3
+    SCHEMA = ListOffsetsRequest_v2.SCHEMA
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+
+class ListOffsetsRequest_v4(Request):
+    """
+    Add current_leader_epoch to request
+    """
+    API_KEY = 2
+    API_VERSION = 4
+    RESPONSE_TYPE = ListOffsetsResponse_v4
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('isolation_level', Int8),  # <- added isolation_level
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('timestamp', Int64)))))
+    )
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+
+class ListOffsetsRequest_v5(Request):
+    API_KEY = 2
+    API_VERSION = 5
+    RESPONSE_TYPE = ListOffsetsResponse_v5
+    SCHEMA = ListOffsetsRequest_v4.SCHEMA
+    DEFAULTS = {
+        'replica_id': -1
+    }
+
+
+ListOffsetsRequest = [
+    ListOffsetsRequest_v0, ListOffsetsRequest_v1, ListOffsetsRequest_v2,
+    ListOffsetsRequest_v3, ListOffsetsRequest_v4, ListOffsetsRequest_v5,
+]
+ListOffsetsResponse = [
+    ListOffsetsResponse_v0, ListOffsetsResponse_v1, ListOffsetsResponse_v2,
+    ListOffsetsResponse_v3, ListOffsetsResponse_v4, ListOffsetsResponse_v5,
+]

venv/lib/python3.11/site-packages/kafka/protocol/message.py

@@ -0,0 +1,216 @@
+from __future__ import absolute_import
+
+import io
+import time
+
+from kafka.codec import (has_gzip, has_snappy, has_lz4, has_zstd,
+                     gzip_decode, snappy_decode, zstd_decode,
+                     lz4_decode, lz4_decode_old_kafka)
+from kafka.protocol.frame import KafkaBytes
+from kafka.protocol.struct import Struct
+from kafka.protocol.types import (
+    Int8, Int32, Int64, Bytes, Schema, AbstractType
+)
+from kafka.util import crc32, WeakMethod
+
+
+class Message(Struct):
+    SCHEMAS = [
+        Schema(
+            ('crc', Int32),
+            ('magic', Int8),
+            ('attributes', Int8),
+            ('key', Bytes),
+            ('value', Bytes)),
+        Schema(
+            ('crc', Int32),
+            ('magic', Int8),
+            ('attributes', Int8),
+            ('timestamp', Int64),
+            ('key', Bytes),
+            ('value', Bytes)),
+    ]
+    SCHEMA = SCHEMAS[1]
+    CODEC_MASK = 0x07
+    CODEC_GZIP = 0x01
+    CODEC_SNAPPY = 0x02
+    CODEC_LZ4 = 0x03
+    CODEC_ZSTD = 0x04
+    TIMESTAMP_TYPE_MASK = 0x08
+    HEADER_SIZE = 22  # crc(4), magic(1), attributes(1), timestamp(8), key+value size(4*2)
+
+    def __init__(self, value, key=None, magic=0, attributes=0, crc=0,
+                 timestamp=None):
+        assert value is None or isinstance(value, bytes), 'value must be bytes'
+        assert key is None or isinstance(key, bytes), 'key must be bytes'
+        assert magic > 0 or timestamp is None, 'timestamp not supported in v0'
+
+        # Default timestamp to now for v1 messages
+        if magic > 0 and timestamp is None:
+            timestamp = int(time.time() * 1000)
+        self.timestamp = timestamp
+        self.crc = crc
+        self._validated_crc = None
+        self.magic = magic
+        self.attributes = attributes
+        self.key = key
+        self.value = value
+        self.encode = WeakMethod(self._encode_self)
+
+    @property
+    def timestamp_type(self):
+        """0 for CreateTime; 1 for LogAppendTime; None if unsupported.
+
+        Value is determined by broker; produced messages should always set to 0
+        Requires Kafka >= 0.10 / message version >= 1
+        """
+        if self.magic == 0:
+            return None
+        elif self.attributes & self.TIMESTAMP_TYPE_MASK:
+            return 1
+        else:
+            return 0
+
+    def _encode_self(self, recalc_crc=True):
+        version = self.magic
+        if version == 1:
+            fields = (self.crc, self.magic, self.attributes, self.timestamp, self.key, self.value)
+        elif version == 0:
+            fields = (self.crc, self.magic, self.attributes, self.key, self.value)
+        else:
+            raise ValueError('Unrecognized message version: %s' % (version,))
+        message = Message.SCHEMAS[version].encode(fields)
+        if not recalc_crc:
+            return message
+        self.crc = crc32(message[4:])
+        crc_field = self.SCHEMAS[version].fields[0]
+        return crc_field.encode(self.crc) + message[4:]
+
+    @classmethod
+    def decode(cls, data):
+        _validated_crc = None
+        if isinstance(data, bytes):
+            _validated_crc = crc32(data[4:])
+            data = io.BytesIO(data)
+        # Partial decode required to determine message version
+        base_fields = cls.SCHEMAS[0].fields[0:3]
+        crc, magic, attributes = [field.decode(data) for field in base_fields]
+        remaining = cls.SCHEMAS[magic].fields[3:]
+        fields = [field.decode(data) for field in remaining]
+        if magic == 1:
+            timestamp = fields[0]
+        else:
+            timestamp = None
+        msg = cls(fields[-1], key=fields[-2],
+                  magic=magic, attributes=attributes, crc=crc,
+                  timestamp=timestamp)
+        msg._validated_crc = _validated_crc
+        return msg
+
+    def validate_crc(self):
+        if self._validated_crc is None:
+            raw_msg = self._encode_self(recalc_crc=False)
+            self._validated_crc = crc32(raw_msg[4:])
+        if self.crc == self._validated_crc:
+            return True
+        return False
+
+    def is_compressed(self):
+        return self.attributes & self.CODEC_MASK != 0
+
+    def decompress(self):
+        codec = self.attributes & self.CODEC_MASK
+        assert codec in (self.CODEC_GZIP, self.CODEC_SNAPPY, self.CODEC_LZ4, self.CODEC_ZSTD)
+        if codec == self.CODEC_GZIP:
+            assert has_gzip(), 'Gzip decompression unsupported'
+            raw_bytes = gzip_decode(self.value)
+        elif codec == self.CODEC_SNAPPY:
+            assert has_snappy(), 'Snappy decompression unsupported'
+            raw_bytes = snappy_decode(self.value)
+        elif codec == self.CODEC_LZ4:
+            assert has_lz4(), 'LZ4 decompression unsupported'
+            if self.magic == 0:
+                raw_bytes = lz4_decode_old_kafka(self.value)
+            else:
+                raw_bytes = lz4_decode(self.value)
+        elif codec == self.CODEC_ZSTD:
+            assert has_zstd(), "ZSTD decompression unsupported"
+            raw_bytes = zstd_decode(self.value)
+        else:
+            raise Exception('This should be impossible')
+
+        return MessageSet.decode(raw_bytes, bytes_to_read=len(raw_bytes))
+
+    def __hash__(self):
+        return hash(self._encode_self(recalc_crc=False))
+
+
+class PartialMessage(bytes):
+    def __repr__(self):
+        return 'PartialMessage(%s)' % (self,)
+
+
+class MessageSet(AbstractType):
+    ITEM = Schema(
+        ('offset', Int64),
+        ('message', Bytes)
+    )
+    HEADER_SIZE = 12  # offset + message_size
+
+    @classmethod
+    def encode(cls, items, prepend_size=True):
+        # RecordAccumulator encodes messagesets internally
+        if isinstance(items, (io.BytesIO, KafkaBytes)):
+            size = Int32.decode(items)
+            if prepend_size:
+                # rewind and return all the bytes
+                items.seek(items.tell() - 4)
+                size += 4
+            return items.read(size)
+
+        encoded_values = []
+        for (offset, message) in items:
+            encoded_values.append(Int64.encode(offset))
+            encoded_values.append(Bytes.encode(message))
+        encoded = b''.join(encoded_values)
+        if prepend_size:
+            return Bytes.encode(encoded)
+        else:
+            return encoded
+
+    @classmethod
+    def decode(cls, data, bytes_to_read=None):
+        """Compressed messages should pass in bytes_to_read (via message size)
+        otherwise, we decode from data as Int32
+        """
+        if isinstance(data, bytes):
+            data = io.BytesIO(data)
+        if bytes_to_read is None:
+            bytes_to_read = Int32.decode(data)
+
+        # if FetchRequest max_bytes is smaller than the available message set
+        # the server returns partial data for the final message
+        # So create an internal buffer to avoid over-reading
+        raw = io.BytesIO(data.read(bytes_to_read))
+
+        items = []
+        while bytes_to_read:
+            try:
+                offset = Int64.decode(raw)
+                msg_bytes = Bytes.decode(raw)
+                bytes_to_read -= 8 + 4 + len(msg_bytes)
+                items.append((offset, len(msg_bytes), Message.decode(msg_bytes)))
+            except ValueError:
+                # PartialMessage to signal that max_bytes may be too small
+                items.append((None, None, PartialMessage()))
+                break
+        return items
+
+    @classmethod
+    def repr(cls, messages):
+        if isinstance(messages, (KafkaBytes, io.BytesIO)):
+            offset = messages.tell()
+            decoded = cls.decode(messages)
+            messages.seek(offset)
+            messages = decoded
+        return str([cls.ITEM.repr(m) for m in messages])

venv/lib/python3.11/site-packages/kafka/protocol/metadata.py

@@ -0,0 +1,257 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Boolean, Int16, Int32, Schema, String
+
+
+class MetadataResponse_v0(Response):
+    API_KEY = 3
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32))),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v1(Response):
+    API_KEY = 3
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v2(Response):
+    API_KEY = 3
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),  # <-- Added cluster_id field in v2
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v3(Response):
+    API_KEY = 3
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v4(Response):
+    API_KEY = 3
+    API_VERSION = 4
+    SCHEMA = MetadataResponse_v3.SCHEMA
+
+
+class MetadataResponse_v5(Response):
+    API_KEY = 3
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32)),
+                ('offline_replicas', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v6(Response):
+    """Metadata Request/Response v6 is the same as v5,
+    but on quota violation, brokers send out responses before throttling."""
+    API_KEY = 3
+    API_VERSION = 6
+    SCHEMA = MetadataResponse_v5.SCHEMA
+
+
+class MetadataResponse_v7(Response):
+    """v7 adds per-partition leader_epoch field"""
+    API_KEY = 3
+    API_VERSION = 7
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('topic', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader', Int32),
+                ('leader_epoch', Int32),
+                ('replicas', Array(Int32)),
+                ('isr', Array(Int32)),
+                ('offline_replicas', Array(Int32))))))
+    )
+
+
+class MetadataRequest_v0(Request):
+    API_KEY = 3
+    API_VERSION = 0
+    RESPONSE_TYPE = MetadataResponse_v0
+    SCHEMA = Schema(
+        ('topics', Array(String('utf-8')))
+    )
+    ALL_TOPICS = [] # Empty Array (len 0) for topics returns all topics
+    NO_TOPICS = [] # v0 does not support a 'no topics' request, so we'll just ask for ALL
+
+
+class MetadataRequest_v1(Request):
+    API_KEY = 3
+    API_VERSION = 1
+    RESPONSE_TYPE = MetadataResponse_v1
+    SCHEMA = MetadataRequest_v0.SCHEMA
+    ALL_TOPICS = None # Null Array (len -1) for topics returns all topics
+    NO_TOPICS = [] # Empty array (len 0) for topics returns no topics
+
+
+class MetadataRequest_v2(Request):
+    API_KEY = 3
+    API_VERSION = 2
+    RESPONSE_TYPE = MetadataResponse_v2
+    SCHEMA = MetadataRequest_v1.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v3(Request):
+    API_KEY = 3
+    API_VERSION = 3
+    RESPONSE_TYPE = MetadataResponse_v3
+    SCHEMA = MetadataRequest_v1.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v4(Request):
+    API_KEY = 3
+    API_VERSION = 4
+    RESPONSE_TYPE = MetadataResponse_v4
+    SCHEMA = Schema(
+        ('topics', Array(String('utf-8'))),
+        ('allow_auto_topic_creation', Boolean)
+    )
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v5(Request):
+    """
+    The v5 metadata request is the same as v4.
+    An additional field for offline_replicas has been added to the v5 metadata response
+    """
+    API_KEY = 3
+    API_VERSION = 5
+    RESPONSE_TYPE = MetadataResponse_v5
+    SCHEMA = MetadataRequest_v4.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v6(Request):
+    API_KEY = 3
+    API_VERSION = 6
+    RESPONSE_TYPE = MetadataResponse_v6
+    SCHEMA = MetadataRequest_v5.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v7(Request):
+    API_KEY = 3
+    API_VERSION = 7
+    RESPONSE_TYPE = MetadataResponse_v7
+    SCHEMA = MetadataRequest_v6.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+MetadataRequest = [
+    MetadataRequest_v0, MetadataRequest_v1, MetadataRequest_v2,
+    MetadataRequest_v3, MetadataRequest_v4, MetadataRequest_v5,
+    MetadataRequest_v6, MetadataRequest_v7,
+]
+MetadataResponse = [
+    MetadataResponse_v0, MetadataResponse_v1, MetadataResponse_v2,
+    MetadataResponse_v3, MetadataResponse_v4, MetadataResponse_v5,
+    MetadataResponse_v6, MetadataResponse_v7,
+]

venv/lib/python3.11/site-packages/kafka/protocol/offset_for_leader_epoch.py

@@ -0,0 +1,140 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, CompactArray, CompactString, Int16, Int32, Int64, Schema, String, TaggedFields
+
+
+class OffsetForLeaderEpochResponse_v0(Response):
+    API_KEY = 23
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v1(Response):
+    API_KEY = 23
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v2(Response):
+    API_KEY = 23
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v3(Response):
+    API_KEY = 23
+    API_VERSION = 3
+    SCHEMA = OffsetForLeaderEpochResponse_v2.SCHEMA
+
+
+class OffsetForLeaderEpochResponse_v4(Response):
+    API_KEY = 23
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', CompactArray(
+            ('topic', CompactString('utf-8')),
+            ('partitions', CompactArray(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64),
+                ('tags', TaggedFields))),
+            ('tags', TaggedFields))),
+        ('tags', TaggedFields))
+
+
+class OffsetForLeaderEpochRequest_v0(Request):
+    API_KEY = 23
+    API_VERSION = 0
+    RESPONSE_TYPE = OffsetForLeaderEpochResponse_v0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v1(Request):
+    API_KEY = 23
+    API_VERSION = 1
+    RESPONSE_TYPE = OffsetForLeaderEpochResponse_v1
+    SCHEMA = OffsetForLeaderEpochRequest_v0.SCHEMA
+
+
+class OffsetForLeaderEpochRequest_v2(Request):
+    API_KEY = 23
+    API_VERSION = 2
+    RESPONSE_TYPE = OffsetForLeaderEpochResponse_v2
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v3(Request):
+    API_KEY = 23
+    API_VERSION = 3
+    RESPONSE_TYPE = OffsetForLeaderEpochResponse_v3
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v4(Request):
+    API_KEY = 23
+    API_VERSION = 4
+    RESPONSE_TYPE = OffsetForLeaderEpochResponse_v4
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', CompactArray(
+            ('topic', CompactString('utf-8')),
+            ('partitions', CompactArray(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32),
+                ('tags', TaggedFields))),
+            ('tags', TaggedFields))),
+        ('tags', TaggedFields))
+
+OffsetForLeaderEpochRequest = [
+    OffsetForLeaderEpochRequest_v0, OffsetForLeaderEpochRequest_v1,
+    OffsetForLeaderEpochRequest_v2, OffsetForLeaderEpochRequest_v3,
+    OffsetForLeaderEpochRequest_v4,
+]
+OffsetForLeaderEpochResponse = [
+    OffsetForLeaderEpochResponse_v0, OffsetForLeaderEpochResponse_v1,
+    OffsetForLeaderEpochResponse_v2, OffsetForLeaderEpochResponse_v3,
+    OffsetForLeaderEpochResponse_v4,
+]

venv/lib/python3.11/site-packages/kafka/protocol/parser.py

@@ -0,0 +1,177 @@
+from __future__ import absolute_import
+
+import collections
+import logging
+
+import kafka.errors as Errors
+from kafka.protocol.find_coordinator import FindCoordinatorResponse
+from kafka.protocol.frame import KafkaBytes
+from kafka.protocol.types import Int32, TaggedFields
+from kafka.version import __version__
+
+log = logging.getLogger(__name__)
+
+
+class KafkaProtocol(object):
+    """Manage the kafka network protocol
+
+    Use an instance of KafkaProtocol to manage bytes send/recv'd
+    from a network socket to a broker.
+
+    Arguments:
+        client_id (str): identifier string to be included in each request
+        api_version (tuple): Optional tuple to specify api_version to use.
+            Currently only used to check for 0.8.2 protocol quirks, but
+            may be used for more in the future.
+    """
+    def __init__(self, client_id=None, api_version=None):
+        if client_id is None:
+            client_id = self._gen_client_id()
+        self._client_id = client_id
+        self._api_version = api_version
+        self._correlation_id = 0
+        self._header = KafkaBytes(4)
+        self._rbuffer = None
+        self._receiving = False
+        self.in_flight_requests = collections.deque()
+        self.bytes_to_send = []
+
+    def _next_correlation_id(self):
+        self._correlation_id = (self._correlation_id + 1) % 2**31
+        return self._correlation_id
+
+    def _gen_client_id(self):
+        return 'kafka-python' + __version__
+
+    def send_request(self, request, correlation_id=None):
+        """Encode and queue a kafka api request for sending.
+
+        Arguments:
+            request (object): An un-encoded kafka request.
+            correlation_id (int, optional): Optionally specify an ID to
+                correlate requests with responses. If not provided, an ID will
+                be generated automatically.
+
+        Returns:
+            correlation_id
+        """
+        log.debug('Sending request %s', request)
+        if correlation_id is None:
+            correlation_id = self._next_correlation_id()
+
+        header = request.build_header(correlation_id=correlation_id, client_id=self._client_id)
+        message = b''.join([header.encode(), request.encode()])
+        size = Int32.encode(len(message))
+        data = size + message
+        self.bytes_to_send.append(data)
+        if request.expect_response():
+            ifr = (correlation_id, request)
+            self.in_flight_requests.append(ifr)
+        return correlation_id
+
+    def send_bytes(self):
+        """Retrieve all pending bytes to send on the network"""
+        data = b''.join(self.bytes_to_send)
+        self.bytes_to_send = []
+        return data
+
+    def receive_bytes(self, data):
+        """Process bytes received from the network.
+
+        Arguments:
+            data (bytes): any length bytes received from a network connection
+                to a kafka broker.
+
+        Returns:
+            responses (list of (correlation_id, response)): any/all completed
+                responses, decoded from bytes to python objects.
+
+        Raises:
+             KafkaProtocolError: if the bytes received could not be decoded.
+             CorrelationIdError: if the response does not match the request
+                 correlation id.
+        """
+        i = 0
+        n = len(data)
+        responses = []
+        while i < n:
+
+            # Not receiving is the state of reading the payload header
+            if not self._receiving:
+                bytes_to_read = min(4 - self._header.tell(), n - i)
+                self._header.write(data[i:i+bytes_to_read])
+                i += bytes_to_read
+
+                if self._header.tell() == 4:
+                    self._header.seek(0)
+                    nbytes = Int32.decode(self._header)
+                    # reset buffer and switch state to receiving payload bytes
+                    self._rbuffer = KafkaBytes(nbytes)
+                    self._receiving = True
+                elif self._header.tell() > 4:
+                    raise Errors.KafkaError('this should not happen - are you threading?')
+
+            if self._receiving:
+                total_bytes = len(self._rbuffer)
+                staged_bytes = self._rbuffer.tell()
+                bytes_to_read = min(total_bytes - staged_bytes, n - i)
+                self._rbuffer.write(data[i:i+bytes_to_read])
+                i += bytes_to_read
+
+                staged_bytes = self._rbuffer.tell()
+                if staged_bytes > total_bytes:
+                    raise Errors.KafkaError('Receive buffer has more bytes than expected?')
+
+                if staged_bytes != total_bytes:
+                    break
+
+                self._receiving = False
+                self._rbuffer.seek(0)
+                resp = self._process_response(self._rbuffer)
+                responses.append(resp)
+                self._reset_buffer()
+        return responses
+
+    def _process_response(self, read_buffer):
+        if not self.in_flight_requests:
+            raise Errors.CorrelationIdError('No in-flight-request found for server response')
+        (correlation_id, request) = self.in_flight_requests.popleft()
+        response_type = request.RESPONSE_TYPE
+        response_header = response_type.parse_header(read_buffer)
+        recv_correlation_id = response_header.correlation_id
+        log.debug('Received correlation id: %d', recv_correlation_id)
+        # 0.8.2 quirk
+        if (recv_correlation_id == 0 and
+            correlation_id != 0 and
+            response_type is FindCoordinatorResponse[0] and
+            (self._api_version == (0, 8, 2) or self._api_version is None)):
+            log.warning('Kafka 0.8.2 quirk -- GroupCoordinatorResponse'
+                        ' Correlation ID does not match request. This'
+                        ' should go away once at least one topic has been'
+                        ' initialized on the broker.')
+
+        elif correlation_id != recv_correlation_id:
+            # return or raise?
+            raise Errors.CorrelationIdError(
+                'Correlation IDs do not match: sent %d, recv %d'
+                % (correlation_id, recv_correlation_id))
+
+        # decode response
+        log.debug('Processing response %s', response_type.__name__)
+        try:
+            response = response_type.decode(read_buffer)
+        except ValueError:
+            read_buffer.seek(0)
+            buf = read_buffer.read()
+            log.error('Response %d [ResponseType: %s Request: %s]:'
+                      ' Unable to decode %d-byte buffer: %r',
+                      correlation_id, response_type,
+                      request, len(buf), buf)
+            raise Errors.KafkaProtocolError('Unable to decode response')
+
+        return (correlation_id, response)
+
+    def _reset_buffer(self):
+        self._receiving = False
+        self._header.seek(0)
+        self._rbuffer = None

venv/lib/python3.11/site-packages/kafka/protocol/pickle.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import
+
+try:
+    import copyreg  # pylint: disable=import-error
+except ImportError:
+    import copy_reg as copyreg  # pylint: disable=import-error
+
+import types
+
+
+def _pickle_method(method):
+    try:
+        func_name = method.__func__.__name__
+        obj = method.__self__
+        cls = method.__self__.__class__
+    except AttributeError:
+        func_name = method.im_func.__name__
+        obj = method.im_self
+        cls = method.im_class
+
+    return _unpickle_method, (func_name, obj, cls)
+
+
+def _unpickle_method(func_name, obj, cls):
+    for cls in cls.mro():
+        try:
+            func = cls.__dict__[func_name]
+        except KeyError:
+            pass
+        else:
+            break
+        return func.__get__(obj, cls)
+
+# https://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
+copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)

venv/lib/python3.11/site-packages/kafka/protocol/produce.py

@@ -0,0 +1,234 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Int16, Int32, Int64, String, Array, Schema, Bytes
+
+
+class ProduceResponse_v0(Response):
+    API_KEY = 0
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offset', Int64)))))
+    )
+
+
+class ProduceResponse_v1(Response):
+    API_KEY = 0
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offset', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ProduceResponse_v2(Response):
+    API_KEY = 0
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offset', Int64),
+                ('timestamp', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ProduceResponse_v3(Response):
+    # Adds support for message format v2
+    API_KEY = 0
+    API_VERSION = 3
+    SCHEMA = ProduceResponse_v2.SCHEMA
+
+
+class ProduceResponse_v4(Response):
+    """
+    The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3
+    """
+    API_KEY = 0
+    API_VERSION = 4
+    SCHEMA = ProduceResponse_v3.SCHEMA
+
+
+class ProduceResponse_v5(Response):
+    API_KEY = 0
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offset', Int64),
+                ('timestamp', Int64),
+                ('log_start_offset', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ProduceResponse_v6(Response):
+    """
+    The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+    """
+    API_KEY = 0
+    API_VERSION = 6
+    SCHEMA = ProduceResponse_v5.SCHEMA
+
+
+class ProduceResponse_v7(Response):
+    """
+    V7 bumped up to indicate ZStandard capability. (see KIP-110)
+    """
+    API_KEY = 0
+    API_VERSION = 7
+    SCHEMA = ProduceResponse_v6.SCHEMA
+
+
+class ProduceResponse_v8(Response):
+    """
+    V8 bumped up to add two new fields record_errors offset list and error_message
+    (See KIP-467)
+    """
+    API_KEY = 0
+    API_VERSION = 8
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16),
+                ('offset', Int64),
+                ('timestamp', Int64),
+                ('log_start_offset', Int64)),
+                ('record_errors', (Array(
+                    ('batch_index', Int32),
+                    ('batch_index_error_message', String('utf-8'))
+                 ))),
+                ('error_message', String('utf-8'))
+             ))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ProduceRequest(Request):
+    API_KEY = 0
+
+    def expect_response(self):
+        if self.required_acks == 0: # pylint: disable=no-member
+            return False
+        return True
+
+
+class ProduceRequest_v0(ProduceRequest):
+    API_VERSION = 0
+    RESPONSE_TYPE = ProduceResponse_v0
+    SCHEMA = Schema(
+        ('required_acks', Int16),
+        ('timeout', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('records', Bytes)))))
+    )
+
+
+class ProduceRequest_v1(ProduceRequest):
+    API_VERSION = 1
+    RESPONSE_TYPE = ProduceResponse_v1
+    SCHEMA = ProduceRequest_v0.SCHEMA
+
+
+class ProduceRequest_v2(ProduceRequest):
+    API_VERSION = 2
+    RESPONSE_TYPE = ProduceResponse_v2
+    SCHEMA = ProduceRequest_v1.SCHEMA
+
+
+class ProduceRequest_v3(ProduceRequest):
+    # Adds support for message format v2
+    API_VERSION = 3
+    RESPONSE_TYPE = ProduceResponse_v3
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('required_acks', Int16),
+        ('timeout', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('records', Bytes)))))
+    )
+
+
+class ProduceRequest_v4(ProduceRequest):
+    """
+    The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3
+    """
+    API_VERSION = 4
+    RESPONSE_TYPE = ProduceResponse_v4
+    SCHEMA = ProduceRequest_v3.SCHEMA
+
+
+class ProduceRequest_v5(ProduceRequest):
+    """
+    Same as v4. The version number is bumped since the v5 response includes an additional
+    partition level field: the log_start_offset.
+    """
+    API_VERSION = 5
+    RESPONSE_TYPE = ProduceResponse_v5
+    SCHEMA = ProduceRequest_v4.SCHEMA
+
+
+class ProduceRequest_v6(ProduceRequest):
+    """
+    The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+    """
+    API_VERSION = 6
+    RESPONSE_TYPE = ProduceResponse_v6
+    SCHEMA = ProduceRequest_v5.SCHEMA
+
+
+class ProduceRequest_v7(ProduceRequest):
+    """
+    V7 bumped up to indicate ZStandard capability. (see KIP-110)
+    """
+    API_VERSION = 7
+    RESPONSE_TYPE = ProduceResponse_v7
+    SCHEMA = ProduceRequest_v6.SCHEMA
+
+
+class ProduceRequest_v8(ProduceRequest):
+    """
+    V8 bumped up to add two new fields record_errors offset list and error_message to PartitionResponse
+    (See KIP-467)
+    """
+    API_VERSION = 8
+    RESPONSE_TYPE = ProduceResponse_v8
+    SCHEMA = ProduceRequest_v7.SCHEMA
+
+
+ProduceRequest = [
+    ProduceRequest_v0, ProduceRequest_v1, ProduceRequest_v2,
+    ProduceRequest_v3, ProduceRequest_v4, ProduceRequest_v5,
+    ProduceRequest_v6, ProduceRequest_v7, ProduceRequest_v8,
+]
+ProduceResponse = [
+    ProduceResponse_v0, ProduceResponse_v1, ProduceResponse_v2,
+    ProduceResponse_v3, ProduceResponse_v4, ProduceResponse_v5,
+    ProduceResponse_v6, ProduceResponse_v7, ProduceResponse_v8,
+]

venv/lib/python3.11/site-packages/kafka/protocol/sasl_authenticate.py

@@ -0,0 +1,42 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Bytes, Int16, Int64, Schema, String
+
+
+class SaslAuthenticateResponse_v0(Response):
+    API_KEY = 36
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('error_message', String('utf-8')),
+        ('auth_bytes', Bytes))
+
+
+class SaslAuthenticateResponse_v1(Response):
+    API_KEY = 36
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('error_message', String('utf-8')),
+        ('auth_bytes', Bytes),
+        ('session_lifetime_ms', Int64))
+
+
+class SaslAuthenticateRequest_v0(Request):
+    API_KEY = 36
+    API_VERSION = 0
+    RESPONSE_TYPE = SaslAuthenticateResponse_v0
+    SCHEMA = Schema(
+        ('auth_bytes', Bytes))
+
+
+class SaslAuthenticateRequest_v1(Request):
+    API_KEY = 36
+    API_VERSION = 1
+    RESPONSE_TYPE = SaslAuthenticateResponse_v1
+    SCHEMA = SaslAuthenticateRequest_v0.SCHEMA
+
+
+SaslAuthenticateRequest = [SaslAuthenticateRequest_v0, SaslAuthenticateRequest_v1]
+SaslAuthenticateResponse = [SaslAuthenticateResponse_v0, SaslAuthenticateResponse_v1]

venv/lib/python3.11/site-packages/kafka/protocol/sasl_handshake.py

@@ -0,0 +1,39 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int16, Schema, String
+
+
+class SaslHandshakeResponse_v0(Response):
+    API_KEY = 17
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('enabled_mechanisms', Array(String('utf-8')))
+    )
+
+
+class SaslHandshakeResponse_v1(Response):
+    API_KEY = 17
+    API_VERSION = 1
+    SCHEMA = SaslHandshakeResponse_v0.SCHEMA
+
+
+class SaslHandshakeRequest_v0(Request):
+    API_KEY = 17
+    API_VERSION = 0
+    RESPONSE_TYPE = SaslHandshakeResponse_v0
+    SCHEMA = Schema(
+        ('mechanism', String('utf-8'))
+    )
+
+
+class SaslHandshakeRequest_v1(Request):
+    API_KEY = 17
+    API_VERSION = 1
+    RESPONSE_TYPE = SaslHandshakeResponse_v1
+    SCHEMA = SaslHandshakeRequest_v0.SCHEMA
+
+
+SaslHandshakeRequest = [SaslHandshakeRequest_v0, SaslHandshakeRequest_v1]
+SaslHandshakeResponse = [SaslHandshakeResponse_v0, SaslHandshakeResponse_v1]

venv/lib/python3.11/site-packages/kafka/protocol/struct.py

@@ -0,0 +1,72 @@
+from __future__ import absolute_import
+
+from io import BytesIO
+
+from kafka.protocol.abstract import AbstractType
+from kafka.protocol.types import Schema
+
+from kafka.util import WeakMethod
+
+
+class Struct(AbstractType):
+    SCHEMA = Schema()
+
+    def __init__(self, *args, **kwargs):
+        if len(args) == len(self.SCHEMA.fields):
+            for i, name in enumerate(self.SCHEMA.names):
+                self.__dict__[name] = args[i]
+        elif len(args) > 0:
+            raise ValueError('Args must be empty or mirror schema')
+        else:
+            for name in self.SCHEMA.names:
+                self.__dict__[name] = kwargs.pop(name, None)
+            if kwargs:
+                raise ValueError('Keyword(s) not in schema %s: %s'
+                                 % (list(self.SCHEMA.names),
+                                    ', '.join(kwargs.keys())))
+
+        # overloading encode() to support both class and instance
+        # Without WeakMethod() this creates circular ref, which
+        # causes instances to "leak" to garbage
+        self.encode = WeakMethod(self._encode_self)
+
+
+    @classmethod
+    def encode(cls, item):  # pylint: disable=E0202
+        bits = []
+        for i, field in enumerate(cls.SCHEMA.fields):
+            bits.append(field.encode(item[i]))
+        return b''.join(bits)
+
+    def _encode_self(self):
+        return self.SCHEMA.encode(
+            [self.__dict__[name] for name in self.SCHEMA.names]
+        )
+
+    @classmethod
+    def decode(cls, data):
+        if isinstance(data, bytes):
+            data = BytesIO(data)
+        return cls(*[field.decode(data) for field in cls.SCHEMA.fields])
+
+    def get_item(self, name):
+        if name not in self.SCHEMA.names:
+            raise KeyError("%s is not in the schema" % name)
+        return self.__dict__[name]
+
+    def __repr__(self):
+        key_vals = []
+        for name, field in zip(self.SCHEMA.names, self.SCHEMA.fields):
+            key_vals.append('%s=%s' % (name, field.repr(self.__dict__[name])))
+        return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
+
+    def __hash__(self):
+        return hash(self.encode())
+
+    def __eq__(self, other):
+        if self.SCHEMA != other.SCHEMA:
+            return False
+        for attr in self.SCHEMA.names:
+            if self.__dict__[attr] != other.__dict__[attr]:
+                return False
+        return True

venv/lib/python3.11/site-packages/kafka/protocol/txn_offset_commit.py

@@ -0,0 +1,78 @@
+from __future__ import absolute_import
+
+from kafka.protocol.api import Request, Response
+from kafka.protocol.types import Array, Int16, Int32, Int64, Schema, String
+
+
+class TxnOffsetCommitResponse_v0(Response):
+    API_KEY = 28
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('error_code', Int16))))))
+
+
+class TxnOffsetCommitResponse_v1(Response):
+    API_KEY = 28
+    API_VERSION = 1
+    SCHEMA = TxnOffsetCommitResponse_v0.SCHEMA
+
+
+class TxnOffsetCommitResponse_v2(Response):
+    API_KEY = 28
+    API_VERSION = 2
+    SCHEMA = TxnOffsetCommitResponse_v1.SCHEMA
+
+
+class TxnOffsetCommitRequest_v0(Request):
+    API_KEY = 28
+    API_VERSION = 0
+    RESPONSE_TYPE = TxnOffsetCommitResponse_v0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('group_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('metadata', String('utf-8')))))))
+
+
+class TxnOffsetCommitRequest_v1(Request):
+    API_KEY = 28
+    API_VERSION = 1
+    RESPONSE_TYPE = TxnOffsetCommitResponse_v1
+    SCHEMA = TxnOffsetCommitRequest_v0.SCHEMA
+
+
+class TxnOffsetCommitRequest_v2(Request):
+    API_KEY = 28
+    API_VERSION = 2
+    RESPONSE_TYPE = TxnOffsetCommitResponse_v2
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('group_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('offset', Int64),
+                ('leader_epoch', Int32),
+                ('metadata', String('utf-8')))))))
+
+
+TxnOffsetCommitRequest = [
+    TxnOffsetCommitRequest_v0, TxnOffsetCommitRequest_v1, TxnOffsetCommitRequest_v2, 
+]
+TxnOffsetCommitResponse = [
+    TxnOffsetCommitResponse_v0, TxnOffsetCommitResponse_v1, TxnOffsetCommitResponse_v2, 
+]

venv/lib/python3.11/site-packages/kafka/protocol/types.py

@@ -0,0 +1,365 @@
+from __future__ import absolute_import
+
+import struct
+from struct import error
+
+from kafka.protocol.abstract import AbstractType
+
+
+def _pack(f, value):
+    try:
+        return f(value)
+    except error as e:
+        raise ValueError("Error encountered when attempting to convert value: "
+                        "{!r} to struct format: '{}', hit error: {}"
+                        .format(value, f, e))
+
+
+def _unpack(f, data):
+    try:
+        (value,) = f(data)
+        return value
+    except error as e:
+        raise ValueError("Error encountered when attempting to convert value: "
+                        "{!r} to struct format: '{}', hit error: {}"
+                        .format(data, f, e))
+
+
+class Int8(AbstractType):
+    _pack = struct.Struct('>b').pack
+    _unpack = struct.Struct('>b').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(1))
+
+
+class Int16(AbstractType):
+    _pack = struct.Struct('>h').pack
+    _unpack = struct.Struct('>h').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(2))
+
+
+class Int32(AbstractType):
+    _pack = struct.Struct('>i').pack
+    _unpack = struct.Struct('>i').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(4))
+
+
+class Int64(AbstractType):
+    _pack = struct.Struct('>q').pack
+    _unpack = struct.Struct('>q').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(8))
+
+
+class Float64(AbstractType):
+    _pack = struct.Struct('>d').pack
+    _unpack = struct.Struct('>d').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(8))
+
+
+class String(AbstractType):
+    def __init__(self, encoding='utf-8'):
+        self.encoding = encoding
+
+    def encode(self, value):
+        if value is None:
+            return Int16.encode(-1)
+        value = str(value).encode(self.encoding)
+        return Int16.encode(len(value)) + value
+
+    def decode(self, data):
+        length = Int16.decode(data)
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding string')
+        return value.decode(self.encoding)
+
+
+class Bytes(AbstractType):
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            return Int32.encode(-1)
+        else:
+            return Int32.encode(len(value)) + value
+
+    @classmethod
+    def decode(cls, data):
+        length = Int32.decode(data)
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding Bytes')
+        return value
+
+    @classmethod
+    def repr(cls, value):
+        return repr(value[:100] + b'...' if value is not None and len(value) > 100 else value)
+
+
+class Boolean(AbstractType):
+    _pack = struct.Struct('>?').pack
+    _unpack = struct.Struct('>?').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(1))
+
+
+class Schema(AbstractType):
+    def __init__(self, *fields):
+        if fields:
+            self.names, self.fields = zip(*fields)
+        else:
+            self.names, self.fields = (), ()
+
+    def encode(self, item):
+        if len(item) != len(self.fields):
+            raise ValueError('Item field count does not match Schema')
+        return b''.join([
+            field.encode(item[i])
+            for i, field in enumerate(self.fields)
+        ])
+
+    def decode(self, data):
+        return tuple([field.decode(data) for field in self.fields])
+
+    def __len__(self):
+        return len(self.fields)
+
+    def repr(self, value):
+        key_vals = []
+        try:
+            for i in range(len(self)):
+                try:
+                    field_val = getattr(value, self.names[i])
+                except AttributeError:
+                    field_val = value[i]
+                key_vals.append('%s=%s' % (self.names[i], self.fields[i].repr(field_val)))
+            return '(' + ', '.join(key_vals) + ')'
+        except Exception:
+            return repr(value)
+
+
+class Array(AbstractType):
+    def __init__(self, *array_of):
+        if len(array_of) > 1:
+            self.array_of = Schema(*array_of)
+        elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType) or
+                                     issubclass(array_of[0], AbstractType)):
+            self.array_of = array_of[0]
+        else:
+            raise ValueError('Array instantiated with no array_of type')
+
+    def encode(self, items):
+        if items is None:
+            return Int32.encode(-1)
+        encoded_items = [self.array_of.encode(item) for item in items]
+        return b''.join(
+            [Int32.encode(len(encoded_items))] +
+            encoded_items
+        )
+
+    def decode(self, data):
+        length = Int32.decode(data)
+        if length == -1:
+            return None
+        return [self.array_of.decode(data) for _ in range(length)]
+
+    def repr(self, list_of_items):
+        if list_of_items is None:
+            return 'NULL'
+        return '[' + ', '.join([self.array_of.repr(item) for item in list_of_items]) + ']'
+
+
+class UnsignedVarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value, i = 0, 0
+        while True:
+            b, = struct.unpack('B', data.read(1))
+            if not (b & 0x80):
+                break
+            value |= (b & 0x7f) << i
+            i += 7
+            if i > 28:
+                raise ValueError('Invalid value {}'.format(value))
+        value |= b << i
+        return value
+
+    @classmethod
+    def encode(cls, value):
+        value &= 0xffffffff
+        ret = b''
+        while (value & 0xffffff80) != 0:
+            b = (value & 0x7f) | 0x80
+            ret += struct.pack('B', b)
+            value >>= 7
+        ret += struct.pack('B', value)
+        return ret
+
+
+class VarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value = UnsignedVarInt32.decode(data)
+        return (value >> 1) ^ -(value & 1)
+
+    @classmethod
+    def encode(cls, value):
+        # bring it in line with the java binary repr
+        value &= 0xffffffff
+        return UnsignedVarInt32.encode((value << 1) ^ (value >> 31))
+
+
+class VarInt64(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value, i = 0, 0
+        while True:
+            b = data.read(1)
+            if not (b & 0x80):
+                break
+            value |= (b & 0x7f) << i
+            i += 7
+            if i > 63:
+                raise ValueError('Invalid value {}'.format(value))
+        value |= b << i
+        return (value >> 1) ^ -(value & 1)
+
+    @classmethod
+    def encode(cls, value):
+        # bring it in line with the java binary repr
+        value &= 0xffffffffffffffff
+        v = (value << 1) ^ (value >> 63)
+        ret = b''
+        while (v & 0xffffffffffffff80) != 0:
+            b = (value & 0x7f) | 0x80
+            ret += struct.pack('B', b)
+            v >>= 7
+        ret += struct.pack('B', v)
+        return ret
+
+
+class CompactString(String):
+    def decode(self, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding string')
+        return value.decode(self.encoding)
+
+    def encode(self, value):
+        if value is None:
+            return UnsignedVarInt32.encode(0)
+        value = str(value).encode(self.encoding)
+        return UnsignedVarInt32.encode(len(value) + 1) + value
+
+
+class TaggedFields(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        num_fields = UnsignedVarInt32.decode(data)
+        ret = {}
+        if not num_fields:
+            return ret
+        prev_tag = -1
+        for i in range(num_fields):
+            tag = UnsignedVarInt32.decode(data)
+            if tag <= prev_tag:
+                raise ValueError('Invalid or out-of-order tag {}'.format(tag))
+            prev_tag = tag
+            size = UnsignedVarInt32.decode(data)
+            val = data.read(size)
+            ret[tag] = val
+        return ret
+
+    @classmethod
+    def encode(cls, value):
+        ret = UnsignedVarInt32.encode(len(value))
+        for k, v in value.items():
+            # do we allow for other data types ?? It could get complicated really fast
+            assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
+            assert isinstance(k, int) and k > 0, 'Key {} is not a positive integer'.format(k)
+            ret += UnsignedVarInt32.encode(k)
+            ret += v
+        return ret
+
+
+class CompactBytes(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding Bytes')
+        return value
+
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            return UnsignedVarInt32.encode(0)
+        else:
+            return UnsignedVarInt32.encode(len(value) + 1) + value
+
+
+class CompactArray(Array):
+
+    def encode(self, items):
+        if items is None:
+            return UnsignedVarInt32.encode(0)
+        return b''.join(
+            [UnsignedVarInt32.encode(len(items) + 1)] +
+            [self.array_of.encode(item) for item in items]
+        )
+
+    def decode(self, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length == -1:
+            return None
+        return [self.array_of.decode(data) for _ in range(length)]
+

venv/lib/python3.11/site-packages/kafka/record/__init__.py

@@ -0,0 +1,3 @@
+from kafka.record.memory_records import MemoryRecords, MemoryRecordsBuilder
+
+__all__ = ["MemoryRecords", "MemoryRecordsBuilder"]

venv/lib/python3.11/site-packages/kafka/record/_crc32c.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+#
+# Taken from https://cloud.google.com/appengine/docs/standard/python/refdocs/\
+# modules/google/appengine/api/files/crc32c?hl=ru
+#
+# Copyright 2007 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Implementation of CRC-32C checksumming as in rfc3720 section B.4.
+See https://en.wikipedia.org/wiki/Cyclic_redundancy_check for details on CRC-32C
+This code is a manual python translation of c code generated by
+pycrc 0.7.1 (https://pycrc.org/). Command line used:
+'./pycrc.py --model=crc-32c --generate c --algorithm=table-driven'
+"""
+
+import array
+
+CRC_TABLE = (
+    0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
+    0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
+    0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b,
+    0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,
+    0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b,
+    0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,
+    0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54,
+    0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,
+    0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
+    0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,
+    0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5,
+    0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,
+    0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45,
+    0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,
+    0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a,
+    0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,
+    0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48,
+    0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
+    0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687,
+    0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,
+    0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927,
+    0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,
+    0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8,
+    0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,
+    0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096,
+    0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,
+    0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,
+    0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,
+    0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9,
+    0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,
+    0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36,
+    0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,
+    0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c,
+    0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,
+    0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043,
+    0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
+    0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3,
+    0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,
+    0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c,
+    0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,
+    0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652,
+    0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,
+    0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d,
+    0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,
+    0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,
+    0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,
+    0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2,
+    0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,
+    0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530,
+    0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,
+    0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff,
+    0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,
+    0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f,
+    0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
+    0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90,
+    0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,
+    0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee,
+    0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,
+    0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321,
+    0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,
+    0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81,
+    0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
+    0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
+    0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
+)
+
+CRC_INIT = 0
+_MASK = 0xFFFFFFFF
+
+
+def crc_update(crc, data):
+    """Update CRC-32C checksum with data.
+    Args:
+        crc: 32-bit checksum to update as long.
+        data: byte array, string or iterable over bytes.
+    Returns:
+        32-bit updated CRC-32C as long.
+    """
+    if not isinstance(data, array.array) or data.itemsize != 1:
+        buf = array.array("B", data)
+    else:
+        buf = data
+    crc = crc ^ _MASK
+    for b in buf:
+        table_index = (crc ^ b) & 0xff
+        crc = (CRC_TABLE[table_index] ^ (crc >> 8)) & _MASK
+    return crc ^ _MASK
+
+
+def crc_finalize(crc):
+    """Finalize CRC-32C checksum.
+    This function should be called as last step of crc calculation.
+    Args:
+        crc: 32-bit checksum as long.
+    Returns:
+        finalized 32-bit checksum as long
+    """
+    return crc & _MASK
+
+
+def crc(data):
+    """Compute CRC-32C checksum of the data.
+    Args:
+        data: byte array, string or iterable over bytes.
+    Returns:
+        32-bit CRC-32C checksum of data as long.
+    """
+    return crc_finalize(crc_update(CRC_INIT, data))
+
+
+if __name__ == "__main__":
+    import sys
+    # TODO remove the pylint disable once pylint fixes
+    # https://github.com/PyCQA/pylint/issues/2571
+    data = sys.stdin.read()  # pylint: disable=assignment-from-no-return
+    print(hex(crc(data)))

venv/lib/python3.11/site-packages/kafka/record/abc.py

@@ -0,0 +1,152 @@
+from __future__ import absolute_import
+
+import abc
+
+from kafka.vendor.six import add_metaclass
+
+
+@add_metaclass(abc.ABCMeta)
+class ABCRecord(object):
+    __slots__ = ()
+
+    @abc.abstractproperty
+    def size_in_bytes(self):
+        """ Number of total bytes in record
+        """
+
+    @abc.abstractproperty
+    def offset(self):
+        """ Absolute offset of record
+        """
+
+    @abc.abstractproperty
+    def timestamp(self):
+        """ Epoch milliseconds
+        """
+
+    @abc.abstractproperty
+    def timestamp_type(self):
+        """ CREATE_TIME(0) or APPEND_TIME(1)
+        """
+
+    @abc.abstractproperty
+    def key(self):
+        """ Bytes key or None
+        """
+
+    @abc.abstractproperty
+    def value(self):
+        """ Bytes value or None
+        """
+
+    @abc.abstractproperty
+    def checksum(self):
+        """ Prior to v2 format CRC was contained in every message. This will
+            be the checksum for v0 and v1 and None for v2 and above.
+        """
+
+    @abc.abstractmethod
+    def validate_crc(self):
+        """ Return True if v0/v1 record matches checksum. noop/True for v2 records
+        """
+
+    @abc.abstractproperty
+    def headers(self):
+        """ If supported by version list of key-value tuples, or empty list if
+            not supported by format.
+        """
+
+
+@add_metaclass(abc.ABCMeta)
+class ABCRecordBatchBuilder(object):
+    __slots__ = ()
+
+    @abc.abstractmethod
+    def append(self, offset, timestamp, key, value, headers=None):
+        """ Writes record to internal buffer.
+
+        Arguments:
+            offset (int): Relative offset of record, starting from 0
+            timestamp (int or None): Timestamp in milliseconds since beginning
+                of the epoch (midnight Jan 1, 1970 (UTC)). If omitted, will be
+                set to current time.
+            key (bytes or None): Key of the record
+            value (bytes or None): Value of the record
+            headers (List[Tuple[str, bytes]]): Headers of the record. Header
+                keys can not be ``None``.
+
+        Returns:
+            (bytes, int): Checksum of the written record (or None for v2 and
+                above) and size of the written record.
+        """
+
+    @abc.abstractmethod
+    def size_in_bytes(self, offset, timestamp, key, value, headers):
+        """ Return the expected size change on buffer (uncompressed) if we add
+            this message. This will account for varint size changes and give a
+            reliable size.
+        """
+
+    @abc.abstractmethod
+    def build(self):
+        """ Close for append, compress if needed, write size and header and
+            return a ready to send buffer object.
+
+            Return:
+                bytearray: finished batch, ready to send.
+        """
+
+
+@add_metaclass(abc.ABCMeta)
+class ABCRecordBatch(object):
+    """ For v2 encapsulates a RecordBatch, for v0/v1 a single (maybe
+        compressed) message.
+    """
+    __slots__ = ()
+
+    @abc.abstractmethod
+    def __iter__(self):
+        """ Return iterator over records (ABCRecord instances). Will decompress
+            if needed.
+        """
+
+    @abc.abstractproperty
+    def base_offset(self):
+        """ Return base offset for batch
+        """
+
+    @abc.abstractproperty
+    def size_in_bytes(self):
+        """ Return size of batch in bytes (includes header overhead)
+        """
+
+    @abc.abstractproperty
+    def magic(self):
+        """ Return magic value (0, 1, 2) for batch.
+        """
+
+
+@add_metaclass(abc.ABCMeta)
+class ABCRecords(object):
+    __slots__ = ()
+
+    @abc.abstractmethod
+    def __init__(self, buffer):
+        """ Initialize with bytes-like object conforming to the buffer
+            interface (ie. bytes, bytearray, memoryview etc.).
+        """
+
+    @abc.abstractmethod
+    def size_in_bytes(self):
+        """ Returns the size of inner buffer.
+        """
+
+    @abc.abstractmethod
+    def next_batch(self):
+        """ Return next batch of records (ABCRecordBatch instances).
+        """
+
+    @abc.abstractmethod
+    def has_next(self):
+        """ True if there are more batches to read, False otherwise.
+        """

venv/lib/python3.11/site-packages/kafka/record/default_records.py

@@ -0,0 +1,776 @@
+# See:
+# https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/\
+#    apache/kafka/common/record/DefaultRecordBatch.java
+# https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/\
+#    apache/kafka/common/record/DefaultRecord.java
+
+# RecordBatch and Record implementation for magic 2 and above.
+# The schema is given below:
+
+# RecordBatch =>
+#  BaseOffset => Int64
+#  Length => Int32
+#  PartitionLeaderEpoch => Int32
+#  Magic => Int8
+#  CRC => Uint32
+#  Attributes => Int16
+#  LastOffsetDelta => Int32 // also serves as LastSequenceDelta
+#  FirstTimestamp => Int64
+#  MaxTimestamp => Int64
+#  ProducerId => Int64
+#  ProducerEpoch => Int16
+#  BaseSequence => Int32
+#  Records => [Record]
+
+# Record =>
+#   Length => Varint
+#   Attributes => Int8
+#   TimestampDelta => Varlong
+#   OffsetDelta => Varint
+#   Key => Bytes
+#   Value => Bytes
+#   Headers => [HeaderKey HeaderValue]
+#     HeaderKey => String
+#     HeaderValue => Bytes
+
+# Note that when compression is enabled (see attributes below), the compressed
+# record data is serialized directly following the count of the number of
+# records. (ie Records => [Record], but without length bytes)
+
+# The CRC covers the data from the attributes to the end of the batch (i.e. all
+# the bytes that follow the CRC). It is located after the magic byte, which
+# means that clients must parse the magic byte before deciding how to interpret
+# the bytes between the batch length and the magic byte. The partition leader
+# epoch field is not included in the CRC computation to avoid the need to
+# recompute the CRC when this field is assigned for every batch that is
+# received by the broker. The CRC-32C (Castagnoli) polynomial is used for the
+# computation.
+
+# The current RecordBatch attributes are given below:
+#
+# * Unused (6-15)
+# * Control (5)
+# * Transactional (4)
+# * Timestamp Type (3)
+# * Compression Type (0-2)
+
+import struct
+import time
+from kafka.record.abc import ABCRecord, ABCRecordBatch, ABCRecordBatchBuilder
+from kafka.record.util import (
+    decode_varint, encode_varint, calc_crc32c, size_of_varint
+)
+from kafka.errors import CorruptRecordError, UnsupportedCodecError
+from kafka.codec import (
+    gzip_encode, snappy_encode, lz4_encode, zstd_encode,
+    gzip_decode, snappy_decode, lz4_decode, zstd_decode
+)
+import kafka.codec as codecs
+
+
+class DefaultRecordBase(object):
+
+    __slots__ = ()
+
+    HEADER_STRUCT = struct.Struct(
+        ">q"  # BaseOffset => Int64
+        "i"  # Length => Int32
+        "i"  # PartitionLeaderEpoch => Int32
+        "b"  # Magic => Int8
+        "I"  # CRC => Uint32
+        "h"  # Attributes => Int16
+        "i"  # LastOffsetDelta => Int32 // also serves as LastSequenceDelta
+        "q"  # FirstTimestamp => Int64
+        "q"  # MaxTimestamp => Int64
+        "q"  # ProducerId => Int64
+        "h"  # ProducerEpoch => Int16
+        "i"  # BaseSequence => Int32
+        "i"  # Records count => Int32
+    )
+    # Byte offset in HEADER_STRUCT of attributes field. Used to calculate CRC
+    ATTRIBUTES_OFFSET = struct.calcsize(">qiibI")
+    CRC_OFFSET = struct.calcsize(">qiib")
+    AFTER_LEN_OFFSET = struct.calcsize(">qi")
+
+    CODEC_MASK = 0x07
+    CODEC_NONE = 0x00
+    CODEC_GZIP = 0x01
+    CODEC_SNAPPY = 0x02
+    CODEC_LZ4 = 0x03
+    CODEC_ZSTD = 0x04
+    TIMESTAMP_TYPE_MASK = 0x08
+    TRANSACTIONAL_MASK = 0x10
+    CONTROL_MASK = 0x20
+
+    LOG_APPEND_TIME = 1
+    CREATE_TIME = 0
+    NO_PRODUCER_ID = -1
+    NO_SEQUENCE = -1
+    MAX_INT = 2147483647
+
+    def _assert_has_codec(self, compression_type):
+        if compression_type == self.CODEC_GZIP:
+            checker, name = codecs.has_gzip, "gzip"
+        elif compression_type == self.CODEC_SNAPPY:
+            checker, name = codecs.has_snappy, "snappy"
+        elif compression_type == self.CODEC_LZ4:
+            checker, name = codecs.has_lz4, "lz4"
+        elif compression_type == self.CODEC_ZSTD:
+            checker, name = codecs.has_zstd, "zstd"
+        else:
+            raise UnsupportedCodecError("Unrecognized compression type: %s" % (compression_type,))
+        if not checker():
+            raise UnsupportedCodecError(
+                "Libraries for {} compression codec not found".format(name))
+
+
+class DefaultRecordBatch(DefaultRecordBase, ABCRecordBatch):
+
+    __slots__ = ("_buffer", "_header_data", "_pos", "_num_records",
+                 "_next_record_index", "_decompressed")
+
+    def __init__(self, buffer):
+        self._buffer = bytearray(buffer)
+        self._header_data = self.HEADER_STRUCT.unpack_from(self._buffer)
+        self._pos = self.HEADER_STRUCT.size
+        self._num_records = self._header_data[12]
+        self._next_record_index = 0
+        self._decompressed = False
+
+    @property
+    def base_offset(self):
+        return self._header_data[0]
+
+    @property
+    def size_in_bytes(self):
+        return self._header_data[1] + self.AFTER_LEN_OFFSET
+
+    @property
+    def leader_epoch(self):
+        return self._header_data[2]
+
+    @property
+    def magic(self):
+        return self._header_data[3]
+
+    @property
+    def crc(self):
+        return self._header_data[4]
+
+    @property
+    def attributes(self):
+        return self._header_data[5]
+
+    @property
+    def last_offset_delta(self):
+        return self._header_data[6]
+
+    @property
+    def last_offset(self):
+        return self.base_offset + self.last_offset_delta
+
+    @property
+    def next_offset(self):
+        return self.last_offset + 1
+
+    @property
+    def compression_type(self):
+        return self.attributes & self.CODEC_MASK
+
+    @property
+    def timestamp_type(self):
+        return int(bool(self.attributes & self.TIMESTAMP_TYPE_MASK))
+
+    @property
+    def is_transactional(self):
+        return bool(self.attributes & self.TRANSACTIONAL_MASK)
+
+    @property
+    def is_control_batch(self):
+        return bool(self.attributes & self.CONTROL_MASK)
+
+    @property
+    def first_timestamp(self):
+        return self._header_data[7]
+
+    @property
+    def max_timestamp(self):
+        return self._header_data[8]
+
+    @property
+    def producer_id(self):
+        return self._header_data[9]
+
+    def has_producer_id(self):
+        return self.producer_id > self.NO_PRODUCER_ID
+
+    @property
+    def producer_epoch(self):
+        return self._header_data[10]
+
+    @property
+    def base_sequence(self):
+        return self._header_data[11]
+
+    @property
+    def has_sequence(self):
+        return self._header_data[11] != -1 # NO_SEQUENCE
+
+    @property
+    def last_sequence(self):
+        if self.base_sequence == self.NO_SEQUENCE:
+            return self.NO_SEQUENCE
+        return self._increment_sequence(self.base_sequence, self.last_offset_delta)
+
+    def _increment_sequence(self, base, increment):
+        if base > (self.MAX_INT - increment):
+            return increment - (self.MAX_INT - base) - 1
+        return base + increment
+
+    @property
+    def records_count(self):
+        return self._header_data[12]
+
+    def _maybe_uncompress(self):
+        if not self._decompressed:
+            compression_type = self.compression_type
+            if compression_type != self.CODEC_NONE:
+                self._assert_has_codec(compression_type)
+                data = memoryview(self._buffer)[self._pos:]
+                if compression_type == self.CODEC_GZIP:
+                    uncompressed = gzip_decode(data)
+                if compression_type == self.CODEC_SNAPPY:
+                    uncompressed = snappy_decode(data.tobytes())
+                if compression_type == self.CODEC_LZ4:
+                    uncompressed = lz4_decode(data.tobytes())
+                if compression_type == self.CODEC_ZSTD:
+                    uncompressed = zstd_decode(data.tobytes())
+                self._buffer = bytearray(uncompressed)
+                self._pos = 0
+        self._decompressed = True
+
+    def _read_msg(
+            self,
+            decode_varint=decode_varint):
+        # Record =>
+        #   Length => Varint
+        #   Attributes => Int8
+        #   TimestampDelta => Varlong
+        #   OffsetDelta => Varint
+        #   Key => Bytes
+        #   Value => Bytes
+        #   Headers => [HeaderKey HeaderValue]
+        #     HeaderKey => String
+        #     HeaderValue => Bytes
+
+        buffer = self._buffer
+        pos = self._pos
+        length, pos = decode_varint(buffer, pos)
+        start_pos = pos
+        _, pos = decode_varint(buffer, pos)  # attrs can be skipped for now
+
+        ts_delta, pos = decode_varint(buffer, pos)
+        if self.timestamp_type == self.LOG_APPEND_TIME:
+            timestamp = self.max_timestamp
+        else:
+            timestamp = self.first_timestamp + ts_delta
+
+        offset_delta, pos = decode_varint(buffer, pos)
+        offset = self.base_offset + offset_delta
+
+        key_len, pos = decode_varint(buffer, pos)
+        if key_len >= 0:
+            key = bytes(buffer[pos: pos + key_len])
+            pos += key_len
+        else:
+            key = None
+
+        value_len, pos = decode_varint(buffer, pos)
+        if value_len >= 0:
+            value = bytes(buffer[pos: pos + value_len])
+            pos += value_len
+        else:
+            value = None
+
+        header_count, pos = decode_varint(buffer, pos)
+        if header_count < 0:
+            raise CorruptRecordError("Found invalid number of record "
+                                         "headers {}".format(header_count))
+        headers = []
+        while header_count:
+            # Header key is of type String, that can't be None
+            h_key_len, pos = decode_varint(buffer, pos)
+            if h_key_len < 0:
+                raise CorruptRecordError(
+                    "Invalid negative header key size {}".format(h_key_len))
+            h_key = buffer[pos: pos + h_key_len].decode("utf-8")
+            pos += h_key_len
+
+            # Value is of type NULLABLE_BYTES, so it can be None
+            h_value_len, pos = decode_varint(buffer, pos)
+            if h_value_len >= 0:
+                h_value = bytes(buffer[pos: pos + h_value_len])
+                pos += h_value_len
+            else:
+                h_value = None
+
+            headers.append((h_key, h_value))
+            header_count -= 1
+
+        # validate whether we have read all header bytes in the current record
+        if pos - start_pos != length:
+            raise CorruptRecordError(
+                "Invalid record size: expected to read {} bytes in record "
+                "payload, but instead read {}".format(length, pos - start_pos))
+        self._pos = pos
+
+        if self.is_control_batch:
+            return ControlRecord(
+                length, offset, timestamp, self.timestamp_type, key, value, headers)
+        else:
+            return DefaultRecord(
+                length, offset, timestamp, self.timestamp_type, key, value, headers)
+
+    def __iter__(self):
+        self._maybe_uncompress()
+        return self
+
+    def __next__(self):
+        if self._next_record_index >= self._num_records:
+            if self._pos != len(self._buffer):
+                raise CorruptRecordError(
+                    "{} unconsumed bytes after all records consumed".format(
+                        len(self._buffer) - self._pos))
+            raise StopIteration
+        try:
+            msg = self._read_msg()
+        except (ValueError, IndexError) as err:
+            raise CorruptRecordError(
+                "Found invalid record structure: {!r}".format(err))
+        else:
+            self._next_record_index += 1
+        return msg
+
+    next = __next__
+
+    def validate_crc(self):
+        assert self._decompressed is False, \
+            "Validate should be called before iteration"
+
+        crc = self.crc
+        data_view = memoryview(self._buffer)[self.ATTRIBUTES_OFFSET:]
+        verify_crc = calc_crc32c(data_view.tobytes())
+        return crc == verify_crc
+
+    def __str__(self):
+        return (
+            "DefaultRecordBatch(magic={}, base_offset={}, last_offset_delta={},"
+            " first_timestamp={}, max_timestamp={},"
+            " is_transactional={}, producer_id={}, producer_epoch={}, base_sequence={},"
+            " records_count={})".format(
+                self.magic, self.base_offset, self.last_offset_delta,
+                self.first_timestamp, self.max_timestamp,
+                self.is_transactional, self.producer_id, self.producer_epoch, self.base_sequence,
+                self.records_count))
+
+
+class DefaultRecord(ABCRecord):
+
+    __slots__ = ("_size_in_bytes", "_offset", "_timestamp", "_timestamp_type", "_key", "_value",
+                 "_headers")
+
+    def __init__(self, size_in_bytes, offset, timestamp, timestamp_type, key, value, headers):
+        self._size_in_bytes = size_in_bytes
+        self._offset = offset
+        self._timestamp = timestamp
+        self._timestamp_type = timestamp_type
+        self._key = key
+        self._value = value
+        self._headers = headers
+
+    @property
+    def size_in_bytes(self):
+        return self._size_in_bytes
+
+    @property
+    def offset(self):
+        return self._offset
+
+    @property
+    def timestamp(self):
+        """ Epoch milliseconds
+        """
+        return self._timestamp
+
+    @property
+    def timestamp_type(self):
+        """ CREATE_TIME(0) or APPEND_TIME(1)
+        """
+        return self._timestamp_type
+
+    @property
+    def key(self):
+        """ Bytes key or None
+        """
+        return self._key
+
+    @property
+    def value(self):
+        """ Bytes value or None
+        """
+        return self._value
+
+    @property
+    def headers(self):
+        return self._headers
+
+    @property
+    def checksum(self):
+        return None
+
+    def validate_crc(self):
+        return True
+
+    def __repr__(self):
+        return (
+            "DefaultRecord(offset={!r}, timestamp={!r}, timestamp_type={!r},"
+            " key={!r}, value={!r}, headers={!r})".format(
+                self._offset, self._timestamp, self._timestamp_type,
+                self._key, self._value, self._headers)
+        )
+
+
+class ControlRecord(DefaultRecord):
+    __slots__ = ("_size_in_bytes", "_offset", "_timestamp", "_timestamp_type", "_key", "_value",
+                 "_headers", "_version", "_type")
+
+    KEY_STRUCT = struct.Struct(
+        ">h"  # Current Version => Int16
+        "h"  # Type => Int16 (0 indicates an abort marker, 1 indicates a commit)
+    )
+
+    def __init__(self, size_in_bytes, offset, timestamp, timestamp_type, key, value, headers):
+        super(ControlRecord, self).__init__(size_in_bytes, offset, timestamp, timestamp_type, key, value, headers)
+        (self._version, self._type) = self.KEY_STRUCT.unpack(self._key)
+
+    # see https://kafka.apache.org/documentation/#controlbatch
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def type(self):
+        return self._type
+
+    @property
+    def abort(self):
+        return self._type == 0
+
+    @property
+    def commit(self):
+        return self._type == 1
+
+    def __repr__(self):
+        return (
+            "ControlRecord(offset={!r}, timestamp={!r}, timestamp_type={!r},"
+            " version={!r}, type={!r} <{!s}>)".format(
+                self._offset, self._timestamp, self._timestamp_type,
+                self._version, self._type, "abort" if self.abort else "commit")
+        )
+
+
+class DefaultRecordBatchBuilder(DefaultRecordBase, ABCRecordBatchBuilder):
+
+    # excluding key, value and headers:
+    # 5 bytes length + 10 bytes timestamp + 5 bytes offset + 1 byte attributes
+    MAX_RECORD_OVERHEAD = 21
+
+    __slots__ = ("_magic", "_compression_type", "_batch_size", "_is_transactional",
+                 "_producer_id", "_producer_epoch", "_base_sequence",
+                 "_first_timestamp", "_max_timestamp", "_last_offset", "_num_records",
+                 "_buffer")
+
+    def __init__(
+            self, magic, compression_type, is_transactional,
+            producer_id, producer_epoch, base_sequence, batch_size):
+        assert magic >= 2
+        self._magic = magic
+        self._compression_type = compression_type & self.CODEC_MASK
+        self._batch_size = batch_size
+        self._is_transactional = bool(is_transactional)
+        # KIP-98 fields for EOS
+        self._producer_id = producer_id
+        self._producer_epoch = producer_epoch
+        self._base_sequence = base_sequence
+
+        self._first_timestamp = None
+        self._max_timestamp = None
+        self._last_offset = 0
+        self._num_records = 0
+
+        self._buffer = bytearray(self.HEADER_STRUCT.size)
+
+    def set_producer_state(self, producer_id, producer_epoch, base_sequence, is_transactional):
+        assert not is_transactional or producer_id != -1, "Cannot write transactional messages without a valid producer ID"
+        assert producer_id == -1 or producer_epoch != -1, "Invalid negative producer epoch"
+        assert producer_id == -1 or base_sequence != -1, "Invalid negative sequence number"
+        self._producer_id = producer_id
+        self._producer_epoch = producer_epoch
+        self._base_sequence = base_sequence
+        self._is_transactional = is_transactional
+
+    @property
+    def producer_id(self):
+        return self._producer_id
+
+    @property
+    def producer_epoch(self):
+        return self._producer_epoch
+
+    def _get_attributes(self, include_compression_type=True):
+        attrs = 0
+        if include_compression_type:
+            attrs |= self._compression_type
+        # Timestamp Type is set by Broker
+        if self._is_transactional:
+            attrs |= self.TRANSACTIONAL_MASK
+        # Control batches are only created by Broker
+        return attrs
+
+    def append(self, offset, timestamp, key, value, headers,
+               # Cache for LOAD_FAST opcodes
+               encode_varint=encode_varint, size_of_varint=size_of_varint,
+               get_type=type, type_int=int, time_time=time.time,
+               byte_like=(bytes, bytearray, memoryview),
+               bytearray_type=bytearray, len_func=len, zero_len_varint=1
+               ):
+        """ Write message to messageset buffer with MsgVersion 2
+        """
+        # Check types
+        if get_type(offset) != type_int:
+            raise TypeError(offset)
+        if timestamp is None:
+            timestamp = type_int(time_time() * 1000)
+        elif get_type(timestamp) != type_int:
+            raise TypeError(timestamp)
+        if not (key is None or get_type(key) in byte_like):
+            raise TypeError(
+                "Not supported type for key: {}".format(type(key)))
+        if not (value is None or get_type(value) in byte_like):
+            raise TypeError(
+                "Not supported type for value: {}".format(type(value)))
+
+        # We will always add the first message, so those will be set
+        if self._first_timestamp is None:
+            self._first_timestamp = timestamp
+            self._max_timestamp = timestamp
+            timestamp_delta = 0
+            first_message = 1
+        else:
+            timestamp_delta = timestamp - self._first_timestamp
+            first_message = 0
+
+        # We can't write record right away to out buffer, we need to
+        # precompute the length as first value...
+        message_buffer = bytearray_type(b"\x00")  # Attributes
+        write_byte = message_buffer.append
+        write = message_buffer.extend
+
+        encode_varint(timestamp_delta, write_byte)
+        # Base offset is always 0 on Produce
+        encode_varint(offset, write_byte)
+
+        if key is not None:
+            encode_varint(len_func(key), write_byte)
+            write(key)
+        else:
+            write_byte(zero_len_varint)
+
+        if value is not None:
+            encode_varint(len_func(value), write_byte)
+            write(value)
+        else:
+            write_byte(zero_len_varint)
+
+        encode_varint(len_func(headers), write_byte)
+
+        for h_key, h_value in headers:
+            h_key = h_key.encode("utf-8")
+            encode_varint(len_func(h_key), write_byte)
+            write(h_key)
+            if h_value is not None:
+                encode_varint(len_func(h_value), write_byte)
+                write(h_value)
+            else:
+                write_byte(zero_len_varint)
+
+        message_len = len_func(message_buffer)
+        main_buffer = self._buffer
+
+        required_size = message_len + size_of_varint(message_len)
+        # Check if we can write this message
+        if (required_size + len_func(main_buffer) > self._batch_size and
+                not first_message):
+            return None
+
+        # Those should be updated after the length check
+        if self._max_timestamp < timestamp:
+            self._max_timestamp = timestamp
+        self._num_records += 1
+        self._last_offset = offset
+
+        encode_varint(message_len, main_buffer.append)
+        main_buffer.extend(message_buffer)
+
+        return DefaultRecordMetadata(offset, required_size, timestamp)
+
+    def write_header(self, use_compression_type=True):
+        batch_len = len(self._buffer)
+        self.HEADER_STRUCT.pack_into(
+            self._buffer, 0,
+            0,  # BaseOffset, set by broker
+            batch_len - self.AFTER_LEN_OFFSET,  # Size from here to end
+            0,  # PartitionLeaderEpoch, set by broker
+            self._magic,
+            0,  # CRC will be set below, as we need a filled buffer for it
+            self._get_attributes(use_compression_type),
+            self._last_offset,
+            self._first_timestamp or 0,
+            self._max_timestamp or 0,
+            self._producer_id,
+            self._producer_epoch,
+            self._base_sequence,
+            self._num_records
+        )
+        crc = calc_crc32c(self._buffer[self.ATTRIBUTES_OFFSET:])
+        struct.pack_into(">I", self._buffer, self.CRC_OFFSET, crc)
+
+    def _maybe_compress(self):
+        if self._compression_type != self.CODEC_NONE:
+            self._assert_has_codec(self._compression_type)
+            header_size = self.HEADER_STRUCT.size
+            data = bytes(self._buffer[header_size:])
+            if self._compression_type == self.CODEC_GZIP:
+                compressed = gzip_encode(data)
+            elif self._compression_type == self.CODEC_SNAPPY:
+                compressed = snappy_encode(data)
+            elif self._compression_type == self.CODEC_LZ4:
+                compressed = lz4_encode(data)
+            elif self._compression_type == self.CODEC_ZSTD:
+                compressed = zstd_encode(data)
+            compressed_size = len(compressed)
+            if len(data) <= compressed_size:
+                # We did not get any benefit from compression, lets send
+                # uncompressed
+                return False
+            else:
+                # Trim bytearray to the required size
+                needed_size = header_size + compressed_size
+                del self._buffer[needed_size:]
+                self._buffer[header_size:needed_size] = compressed
+                return True
+        return False
+
+    def build(self):
+        send_compressed = self._maybe_compress()
+        self.write_header(send_compressed)
+        return self._buffer
+
+    def size(self):
+        """ Return current size of data written to buffer
+        """
+        return len(self._buffer)
+
+    @classmethod
+    def header_size_in_bytes(self):
+        return self.HEADER_STRUCT.size
+
+    @classmethod
+    def size_in_bytes(self, offset_delta, timestamp_delta, key, value, headers):
+        size_of_body = (
+            1 +  # Attrs
+            size_of_varint(offset_delta) +
+            size_of_varint(timestamp_delta) +
+            self.size_of(key, value, headers)
+        )
+        return size_of_body + size_of_varint(size_of_body)
+
+    @classmethod
+    def size_of(cls, key, value, headers):
+        size = 0
+        # Key size
+        if key is None:
+            size += 1
+        else:
+            key_len = len(key)
+            size += size_of_varint(key_len) + key_len
+        # Value size
+        if value is None:
+            size += 1
+        else:
+            value_len = len(value)
+            size += size_of_varint(value_len) + value_len
+        # Header size
+        size += size_of_varint(len(headers))
+        for h_key, h_value in headers:
+            h_key_len = len(h_key.encode("utf-8"))
+            size += size_of_varint(h_key_len) + h_key_len
+
+            if h_value is None:
+                size += 1
+            else:
+                h_value_len = len(h_value)
+                size += size_of_varint(h_value_len) + h_value_len
+        return size
+
+    @classmethod
+    def estimate_size_in_bytes(cls, key, value, headers):
+        """ Get the upper bound estimate on the size of record
+        """
+        return (
+            cls.HEADER_STRUCT.size + cls.MAX_RECORD_OVERHEAD +
+            cls.size_of(key, value, headers)
+        )
+
+    def __str__(self):
+        return (
+            "DefaultRecordBatchBuilder(magic={}, base_offset={}, last_offset_delta={},"
+            " first_timestamp={}, max_timestamp={},"
+            " is_transactional={}, producer_id={}, producer_epoch={}, base_sequence={},"
+            " records_count={})".format(
+                self._magic, 0, self._last_offset,
+                self._first_timestamp or 0, self._max_timestamp or 0,
+                self._is_transactional, self._producer_id, self._producer_epoch, self._base_sequence,
+                self._num_records))
+
+
+class DefaultRecordMetadata(object):
+
+    __slots__ = ("_size", "_timestamp", "_offset")
+
+    def __init__(self, offset, size, timestamp):
+        self._offset = offset
+        self._size = size
+        self._timestamp = timestamp
+
+    @property
+    def offset(self):
+        return self._offset
+
+    @property
+    def crc(self):
+        return None
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def timestamp(self):
+        return self._timestamp
+
+    def __repr__(self):
+        return (
+            "DefaultRecordMetadata(offset={!r}, size={!r}, timestamp={!r})"
+            .format(self._offset, self._size, self._timestamp)
+        )