From c4bde06838b871492dba0a4c0026a856dab1fcae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mr=C2=B7x?= Date: Fri, 9 Oct 2020 10:58:37 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Kafka/Broker.php | 246 +++++++++ Kafka/Config.php | 193 +++++++ Kafka/Consumer.php | 83 +++ Kafka/Consumer/Assignment.php | 301 +++++++++++ Kafka/Consumer/Process.php | 812 ++++++++++++++++++++++++++++ Kafka/Consumer/State.php | 457 ++++++++++++++++ Kafka/ConsumerConfig.php | 149 +++++ Kafka/Exception.php | 31 ++ Kafka/Exception/Config.php | 33 ++ Kafka/Exception/NotSupported.php | 33 ++ Kafka/Exception/Protocol.php | 33 ++ Kafka/Exception/Socket.php | 9 + Kafka/LoggerTrait.php | 165 ++++++ Kafka/Producer.php | 129 +++++ Kafka/Producer/Process.php | 384 +++++++++++++ Kafka/Producer/State.php | 275 ++++++++++ Kafka/Producer/SyncProcess.php | 200 +++++++ Kafka/ProducerConfig.php | 102 ++++ Kafka/Protocol.php | 294 ++++++++++ Kafka/Protocol/CommitOffset.php | 226 ++++++++ Kafka/Protocol/DescribeGroups.php | 201 +++++++ Kafka/Protocol/Fetch.php | 377 +++++++++++++ Kafka/Protocol/FetchOffset.php | 185 +++++++ Kafka/Protocol/GroupCoordinator.php | 85 +++ Kafka/Protocol/Heartbeat.php | 85 +++ Kafka/Protocol/JoinGroup.php | 204 +++++++ Kafka/Protocol/LeaveGroup.php | 79 +++ Kafka/Protocol/ListGroup.php | 99 ++++ Kafka/Protocol/Metadata.php | 179 ++++++ Kafka/Protocol/Offset.php | 203 +++++++ Kafka/Protocol/Produce.php | 276 ++++++++++ Kafka/Protocol/Protocol.php | 679 +++++++++++++++++++++++ Kafka/Protocol/SyncGroup.php | 212 ++++++++ Kafka/SingletonTrait.php | 76 +++ Kafka/Socket.php | 427 +++++++++++++++ Kafka/SocketSync.php | 394 ++++++++++++++ composer.json | 6 +- 37 files changed, 7920 insertions(+), 2 deletions(-) create mode 100644 Kafka/Broker.php create mode 100644 Kafka/Config.php create mode 100644 Kafka/Consumer.php create mode 100644 Kafka/Consumer/Assignment.php create mode 100644 Kafka/Consumer/Process.php create mode 100644 Kafka/Consumer/State.php create mode 100644 Kafka/ConsumerConfig.php create mode 100644 Kafka/Exception.php create mode 100644 Kafka/Exception/Config.php create mode 100644 Kafka/Exception/NotSupported.php create mode 100644 Kafka/Exception/Protocol.php create mode 100644 Kafka/Exception/Socket.php create mode 100644 Kafka/LoggerTrait.php create mode 100644 Kafka/Producer.php create mode 100644 Kafka/Producer/Process.php create mode 100644 Kafka/Producer/State.php create mode 100644 Kafka/Producer/SyncProcess.php create mode 100644 Kafka/ProducerConfig.php create mode 100644 Kafka/Protocol.php create mode 100644 Kafka/Protocol/CommitOffset.php create mode 100644 Kafka/Protocol/DescribeGroups.php create mode 100644 Kafka/Protocol/Fetch.php create mode 100644 Kafka/Protocol/FetchOffset.php create mode 100644 Kafka/Protocol/GroupCoordinator.php create mode 100644 Kafka/Protocol/Heartbeat.php create mode 100644 Kafka/Protocol/JoinGroup.php create mode 100644 Kafka/Protocol/LeaveGroup.php create mode 100644 Kafka/Protocol/ListGroup.php create mode 100644 Kafka/Protocol/Metadata.php create mode 100644 Kafka/Protocol/Offset.php create mode 100644 Kafka/Protocol/Produce.php create mode 100644 Kafka/Protocol/Protocol.php create mode 100644 Kafka/Protocol/SyncGroup.php create mode 100644 Kafka/SingletonTrait.php create mode 100644 Kafka/Socket.php create mode 100644 Kafka/SocketSync.php diff --git a/Kafka/Broker.php b/Kafka/Broker.php new file mode 100644 index 00000000..afd483cf --- /dev/null +++ b/Kafka/Broker.php @@ -0,0 +1,246 @@ +process = $process; + } + + // }}} + // {{{ public function setGroupBrokerId() + + public function setGroupBrokerId($brokerId) + { + $this->groupBrokerId = $brokerId; + } + + // }}} + // {{{ public function getGroupBrokerId() + + public function getGroupBrokerId() + { + return $this->groupBrokerId; + } + + // }}} + // {{{ public function setData() + + public function setData($topics, $brokersResult) + { + $brokers = array(); + foreach ($brokersResult as $value) { + $key = $value['nodeId']; + $hostname = $value['host'] . ':' . $value['port']; + $brokers[$key] = $hostname; + } + + $change = false; + if (serialize($this->brokers) != serialize($brokers)) { + $this->brokers = $brokers; + $change = true; + } + + $newTopics = array(); + foreach ($topics as $topic) { + if ($topic['errorCode'] != \Kafka\Protocol::NO_ERROR) { + $this->error('Parse metadata for topic is error, error:' . \Kafka\Protocol::getError($topic['errorCode'])); + continue; + } + $item = array(); + foreach ($topic['partitions'] as $part) { + $item[$part['partitionId']] = $part['leader']; + } + $newTopics[$topic['topicName']] = $item; + } + + if (serialize($this->topics) != serialize($newTopics)) { + $this->topics = $newTopics; + $change = true; + } + + return $change; + } + + // }}} + // {{{ public function getTopics() + + public function getTopics() + { + return $this->topics; + } + + // }}} + // {{{ public function getBrokers() + + public function getBrokers() + { + return $this->brokers; + } + + // }}} + // {{{ public function getMetaConnect() + + public function getMetaConnect($key, $modeSync = false) + { + return $this->getConnect($key, 'metaSockets', $modeSync); + } + + // }}} + // {{{ public function getRandConnect() + + public function getRandConnect($modeSync = false) + { + $nodeIds = array_keys($this->brokers); + shuffle($nodeIds); + if (!isset($nodeIds[0])) { + return false; + } + return $this->getMetaConnect($nodeIds[0], $modeSync); + } + + // }}} + // {{{ public function getDataConnect() + + public function getDataConnect($key, $modeSync = false) + { + return $this->getConnect($key, 'dataSockets', $modeSync); + } + + // }}} + // {{{ public function getConnect() + + public function getConnect($key, $type, $modeSync = false) + { + if (isset($this->{$type}[$key])) { + return $this->{$type}[$key]; + } + + if (isset($this->brokers[$key])) { + $hostname = $this->brokers[$key]; + if (isset($this->{$type}[$hostname])) { + return $this->{$type}[$hostname]; + } + } + + $host = null; + $port = null; + if (isset($this->brokers[$key])) { + $hostname = $this->brokers[$key]; + list($host, $port) = explode(':', $hostname); + } + + if (strpos($key, ':') !== false) { + list($host, $port) = explode(':', $key); + } + + if ($host && $port) { + try { + $socket = $this->getSocket($host, $port, $modeSync); + if (!$modeSync) { + $socket->SetonReadable($this->process); + } + $socket->connect(); + $this->{$type}[$key] = $socket; + return $socket; + } catch (\Exception $e) { + $this->error($e->getMessage()); + return false; + } + } else { + return false; + } + } + + // }}} + // {{{ public function clear() + + public function clear() + { + foreach ($this->metaSockets as $key => $socket) { + $socket->close(); + } + foreach ($this->dataSockets as $key => $socket) { + $socket->close(); + } + $this->brokers = []; + } + + // }}} + // {{{ public function getSocket() + + public function getSocket($host, $port, $modeSync) + { + if ($this->socket != null) { + return $this->socket; + } + + if ($modeSync) { + $socket = new \Kafka\SocketSync($host, $port); + } else { + $socket = new \Kafka\Socket($host, $port); + } + return $socket; + } + + // }}} + // {{{ public function setSocket() + + // use unit test + public function setSocket($socket) + { + $this->socket = $socket; + } + + // }}} + // }}} +} diff --git a/Kafka/Config.php b/Kafka/Config.php new file mode 100644 index 00000000..763aa124 --- /dev/null +++ b/Kafka/Config.php @@ -0,0 +1,193 @@ + 'kafka-php', + 'brokerVersion' => '0.10.1.0', + 'metadataBrokerList' => '', + 'messageMaxBytes' => '1000000', + 'metadataRequestTimeoutMs' => '60000', + 'metadataRefreshIntervalMs' => '300000', + 'metadataMaxAgeMs' => -1, + ); + + // }}} + // {{{ functions + // {{{ public function __call() + + public function __call($name, $args) + { + if (strpos($name, 'get') === 0 || strpos($name, 'iet') === 0) { + $option = strtolower(substr($name, 3, 1)) . substr($name, 4); + if (isset(self::$options[$option])) { + return self::$options[$option]; + } + + if (isset(self::$defaults[$option])) { + return self::$defaults[$option]; + } + if (isset(static::$defaults[$option])) { + return static::$defaults[$option]; + } + return false; + } + + if (strpos($name, 'set') === 0) { + if (count($args) != 1) { + return false; + } + $option = strtolower(substr($name, 3, 1)) . substr($name, 4); + static::$options[$option] = $args[0]; + // check todo + return true; + } + } + + // }}} + // {{{ public function setClientId() + + public function setClientId($val) + { + $client = trim($val); + if ($client == '') { + throw new \Kafka\Exception\Config('Set clientId value is invalid, must is not empty string.'); + } + static::$options['clientId'] = $client; + } + + // }}} + // {{{ public function setBrokerVersion() + + public function setBrokerVersion($version) + { + $version = trim($version); + if ($version == '' || version_compare($version, '0.8.0') < 0) { + throw new \Kafka\Exception\Config('Set broker version value is invalid, must is not empty string and gt 0.8.0.'); + } + static::$options['brokerVersion'] = $version; + } + + // }}} + // {{{ public function setMetadataBrokerList() + + public function setMetadataBrokerList($list) + { + if (trim($list) == '') { + throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); + } + $tmp = explode(',', trim($list)); + $lists = array(); + foreach ($tmp as $key => $val) { + if (trim($val) != '') { + $lists[] = $val; + } + } + if (empty($lists)) { + throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); + } + foreach ($lists as $val) { + $hostinfo = explode(':', $val); + foreach ($hostinfo as $key => $val) { + if (trim($val) == '') { + unset($hostinfo[$key]); + } + } + if (count($hostinfo) != 2) { + throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); + } + } + + static::$options['metadataBrokerList'] = $list; + } + + // }}} + // {{{ public function clear() + + public function clear() + { + static::$options = []; + } + + // }}} + // {{{ public function setMessageMaxBytes() + + public function setMessageMaxBytes($messageMaxBytes) + { + if (!is_numeric($messageMaxBytes) || $messageMaxBytes < 1000 || $messageMaxBytes > 1000000000) { + throw new \Kafka\Exception\Config('Set message max bytes value is invalid, must set it 1000 .. 1000000000'); + } + static::$options['messageMaxBytes'] = $messageMaxBytes; + } + + // }}} + // {{{ public function setMetadataRequestTimeoutMs() + + public function setMetadataRequestTimeoutMs($metadataRequestTimeoutMs) + { + if (!is_numeric($metadataRequestTimeoutMs) || $metadataRequestTimeoutMs < 10 + || $metadataRequestTimeoutMs > 900000) { + throw new \Kafka\Exception\Config('Set metadata request timeout value is invalid, must set it 10 .. 900000'); + } + static::$options['metadataRequestTimeoutMs'] = $metadataRequestTimeoutMs; + } + + // }}} + // {{{ public function setMetadataRefreshIntervalMs() + + public function setMetadataRefreshIntervalMs($metadataRefreshIntervalMs) + { + if (!is_numeric($metadataRefreshIntervalMs) || $metadataRefreshIntervalMs < 10 + || $metadataRefreshIntervalMs > 3600000) { + throw new \Kafka\Exception\Config('Set metadata refresh interval value is invalid, must set it 10 .. 3600000'); + } + static::$options['metadataRefreshIntervalMs'] = $metadataRefreshIntervalMs; + } + + // }}} + // {{{ public function setMetadataMaxAgeMs() + + public function setMetadataMaxAgeMs($metadataMaxAgeMs) + { + if (!is_numeric($metadataMaxAgeMs) || $metadataMaxAgeMs < 1 + || $metadataMaxAgeMs > 86400000) { + throw new \Kafka\Exception\Config('Set metadata max age value is invalid, must set it 1 .. 86400000'); + } + static::$options['metadataMaxAgeMs'] = $metadataMaxAgeMs; + } + + // }}} + // }}} +} diff --git a/Kafka/Consumer.php b/Kafka/Consumer.php new file mode 100644 index 00000000..faa1a635 --- /dev/null +++ b/Kafka/Consumer.php @@ -0,0 +1,83 @@ +isRunning) { + $this->error('Has start consumer'); + return; + } + $process = new \Kafka\Consumer\Process($consumer); + if ($this->logger) { + $process->setLogger($this->logger); + } + $process->start(); + $this->isRunning = true; + if ($isBlock) { + \Amp\run(); + } + } + + // }}} + // }}} +} diff --git a/Kafka/Consumer/Assignment.php b/Kafka/Consumer/Assignment.php new file mode 100644 index 00000000..1165437c --- /dev/null +++ b/Kafka/Consumer/Assignment.php @@ -0,0 +1,301 @@ +memberId = $memberId; + } + + // }}} + // {{{ public function getMemberId() + + public function getMemberId() + { + return $this->memberId; + } + + // }}} + // {{{ public function setGenerationId() + + public function setGenerationId($generationId) + { + $this->generationId = $generationId; + } + + // }}} + // {{{ public function getGenerationId() + + public function getGenerationId() + { + return $this->generationId; + } + + // }}} + // {{{ public function getAssignments() + + public function getAssignments() + { + return $this->assignments; + } + + // }}} + // {{{ public function assign() + + public function assign($result) + { + $broker = \Kafka\Broker::getInstance(); + $topics = $broker->getTopics(); + + $memberCount = count($result); + + $count = 0; + $members = array(); + foreach ($topics as $topicName => $partition) { + foreach ($partition as $partId => $leaderId) { + $memberNum = $count % $memberCount; + if (!isset($members[$memberNum])) { + $members[$memberNum] = array(); + } + if (!isset($members[$memberNum][$topicName])) { + $members[$memberNum][$topicName] = array(); + } + $members[$memberNum][$topicName]['topic_name'] = $topicName; + if (!isset($members[$memberNum][$topicName]['partitions'])) { + $members[$memberNum][$topicName]['partitions'] = array(); + } + $members[$memberNum][$topicName]['partitions'][] = $partId; + $count++; + } + } + + $data = array(); + foreach ($result as $key => $member) { + $item = array( + 'version' => 0, + 'member_id' => $member['memberId'], + 'assignments' => isset($members[$key]) ? $members[$key] : array() + ); + $data[] = $item; + } + $this->assignments = $data; + } + + // }}} + // {{{ public function setTopics() + + public function setTopics($topics) + { + $this->topics = $topics; + } + + // }}} + // {{{ public function getTopics() + + public function getTopics() + { + return $this->topics; + } + + // }}} + // {{{ public function setOffsets() + + public function setOffsets($offsets) + { + $this->offsets = $offsets; + } + + // }}} + // {{{ public function getOffsets() + + public function getOffsets() + { + return $this->offsets; + } + + // }}} + // {{{ public function setLastOffsets() + + public function setLastOffsets($offsets) + { + $this->lastOffsets = $offsets; + } + + // }}} + // {{{ public function getOffsets() + + public function getLastOffsets() + { + return $this->lastOffsets; + } + + // }}} + // {{{ public function setFetchOffsets() + + public function setFetchOffsets($offsets) + { + $this->fetchOffsets = $offsets; + } + + // }}} + // {{{ public function getFetchOffsets() + + public function getFetchOffsets() + { + return $this->fetchOffsets; + } + + // }}} + // {{{ public function setConsumerOffsets() + + public function setConsumerOffsets($offsets) + { + $this->consumerOffsets = $offsets; + } + + // }}} + // {{{ public function getConsumerOffsets() + + public function getConsumerOffsets() + { + return $this->consumerOffsets; + } + + // }}} + // {{{ public function setConsumerOffset() + + public function setConsumerOffset($topic, $part, $offset) + { + $this->consumerOffsets[$topic][$part] = $offset; + } + + // }}} + // {{{ public function getConsumerOffset() + + public function getConsumerOffset($topic, $part) + { + if (!isset($this->consumerOffsets[$topic][$part])) { + return false; + } + return $this->consumerOffsets[$topic][$part]; + } + + // }}} + // {{{ public function setCommitOffsets() + + public function setCommitOffsets($offsets) + { + $this->commitOffsets = $offsets; + } + + // }}} + // {{{ public function getCommitOffsets() + + public function getCommitOffsets() + { + return $this->commitOffsets; + } + + // }}} + // {{{ public function setCommitOffset() + + public function setCommitOffset($topic, $part, $offset) + { + $this->commitOffsets[$topic][$part] = $offset; + } + + // }}} + // {{{ public function setPrecommitOffsets() + + public function setPrecommitOffsets($offsets) + { + $this->precommitOffsets = $offsets; + } + + // }}} + // {{{ public function getPrecommitOffsets() + + public function getPrecommitOffsets() + { + return $this->precommitOffsets; + } + + // }}} + // {{{ public function setPrecommitOffset() + + public function setPrecommitOffset($topic, $part, $offset) + { + $this->precommitOffsets[$topic][$part] = $offset; + } + + // }}} + // {{{ public function clearOffset() + + public function clearOffset() + { + $this->offsets = array(); + $this->lastOffsets = array(); + $this->fetchOffsets = array(); + $this->consumerOffsets = array(); + $this->commitOffsets = array(); + $this->precommitOffsets = array(); + } + + // }}} + // }}} +} diff --git a/Kafka/Consumer/Process.php b/Kafka/Consumer/Process.php new file mode 100644 index 00000000..bb5ff167 --- /dev/null +++ b/Kafka/Consumer/Process.php @@ -0,0 +1,812 @@ +consumer = $consumer; + } + + // }}} + // {{{ public function init() + + /** + * start consumer + * + * @access public + * @return void + */ + public function init() + { + // init protocol + $config = \Kafka\ConsumerConfig::getInstance(); + \Kafka\Protocol::init($config->getBrokerVersion(), $this->logger); + + // init process request + $broker = \Kafka\Broker::getInstance(); + $broker->setProcess(function ($data, $fd) { + $this->processRequest($data, $fd); + }); + + // init state + $this->state = \Kafka\Consumer\State::getInstance(); + if ($this->logger) { + $this->state->setLogger($this->logger); + } + $this->state->setCallback(array( + \Kafka\Consumer\State::REQUEST_METADATA => function () { + return $this->syncMeta(); + }, + \Kafka\Consumer\State::REQUEST_GETGROUP => function () { + return $this->getGroupBrokerId(); + }, + \Kafka\Consumer\State::REQUEST_JOINGROUP => function () { + return $this->joinGroup(); + }, + \Kafka\Consumer\State::REQUEST_SYNCGROUP => function () { + return $this->syncGroup(); + }, + \Kafka\Consumer\State::REQUEST_HEARTGROUP => function () { + return $this->heartbeat(); + }, + \Kafka\Consumer\State::REQUEST_OFFSET => function () { + return $this->offset(); + }, + \Kafka\Consumer\State::REQUEST_FETCH_OFFSET => function () { + return $this->fetchOffset(); + }, + \Kafka\Consumer\State::REQUEST_FETCH => function () { + return $this->fetch(); + }, + \Kafka\Consumer\State::REQUEST_COMMIT_OFFSET => function () { + return $this->commit(); + }, + )); + $this->state->init(); + } + + // }}} + // {{{ public function start() + + /** + * start consumer + * + * @access public + * @return void + */ + public function start() + { + $this->init(); + $this->state->start(); + } + + // }}} + // {{{ public function stop() + + /** + * stop consumer + * + * @access public + * @return void + */ + public function stop() + { + $this->isRunning = false; + } + + // }}} + // {{{ protected function processRequest() + + /** + * process Request + * + * @access public + * @return void + */ + protected function processRequest($data, $fd) + { + $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); + switch ($correlationId) { + case \Kafka\Protocol::METADATA_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); + if (!isset($result['brokers']) || !isset($result['topics'])) { + $this->error('Get metadata is fail, brokers or topics is null.'); + $this->state->failRun(\Kafka\Consumer\State::REQUEST_METADATA); + } else { + $broker = \Kafka\Broker::getInstance(); + $isChange = $broker->setData($result['topics'], $result['brokers']); + $this->state->succRun(\Kafka\Consumer\State::REQUEST_METADATA, $isChange); + } + break; + case \Kafka\Protocol::GROUP_COORDINATOR_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::GROUP_COORDINATOR_REQUEST, substr($data, 4)); + if (isset($result['errorCode']) && $result['errorCode'] == \Kafka\Protocol::NO_ERROR + && isset($result['coordinatorId'])) { + \Kafka\Broker::getInstance()->setGroupBrokerId($result['coordinatorId']); + $this->state->succRun(\Kafka\Consumer\State::REQUEST_GETGROUP); + } else { + $this->state->failRun(\Kafka\Consumer\State::REQUEST_GETGROUP); + } + break; + case \Kafka\Protocol::JOIN_GROUP_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::JOIN_GROUP_REQUEST, substr($data, 4)); + if (isset($result['errorCode']) && $result['errorCode'] == 0) { + $this->succJoinGroup($result); + } else { + $this->failJoinGroup($result['errorCode']); + } + break; + case \Kafka\Protocol::SYNC_GROUP_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::SYNC_GROUP_REQUEST, substr($data, 4)); + if (isset($result['errorCode']) && $result['errorCode'] == 0) { + $this->succSyncGroup($result); + } else { + $this->failSyncGroup($result['errorCode']); + } + break; + case \Kafka\Protocol::HEART_BEAT_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::HEART_BEAT_REQUEST, substr($data, 4)); + if (isset($result['errorCode']) && $result['errorCode'] == 0) { + $this->state->succRun(\Kafka\Consumer\State::REQUEST_HEARTGROUP); + } else { + $this->failHeartbeat($result['errorCode']); + } + break; + case \Kafka\Protocol::OFFSET_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_REQUEST, substr($data, 4)); + $this->succOffset($result, $fd); + break; + case \Kafka\Protocol\Protocol::OFFSET_FETCH_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_FETCH_REQUEST, substr($data, 4)); + $this->succFetchOffset($result); + break; + case \Kafka\Protocol\Protocol::FETCH_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::FETCH_REQUEST, substr($data, 4)); + $this->succFetch($result, $fd); + break; + case \Kafka\Protocol\Protocol::OFFSET_COMMIT_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_COMMIT_REQUEST, substr($data, 4)); + $this->succCommit($result); + break; + default: + $this->error('Error request, correlationId:' . $correlationId); + } + } + + // }}} + // {{{ protected function syncMeta() + + protected function syncMeta() + { + $this->debug('Start sync metadata request'); + $brokerList = explode(',', \Kafka\ConsumerConfig::getInstance()->getMetadataBrokerList()); + $brokerHost = array(); + foreach ($brokerList as $key => $val) { + if (trim($val)) { + $brokerHost[] = $val; + } + } + if (count($brokerHost) == 0) { + throw new \Kafka\Exception('Not set config `metadataBrokerList`'); + } + shuffle($brokerHost); + $broker = \Kafka\Broker::getInstance(); + foreach ($brokerHost as $host) { + $socket = $broker->getMetaConnect($host); + if ($socket) { + $params = \Kafka\ConsumerConfig::getInstance()->getTopics(); + $this->debug('Start sync metadata request params:' . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); + $socket->write($requestData); + return; + } + } + throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); + } + + // }}} + // {{{ protected function getGroupBrokerId() + + protected function getGroupBrokerId() + { + $broker = \Kafka\Broker::getInstance(); + $connect = $broker->getRandConnect(); + if (!$connect) { + return; + } + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + ); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::GROUP_COORDINATOR_REQUEST, $params); + $connect->write($requestData); + } + + // }}} + // {{{ protected function joinGroup() + + protected function joinGroup() + { + $broker = \Kafka\Broker::getInstance(); + $groupBrokerId = $broker->getGroupBrokerId(); + $connect = $broker->getMetaConnect($groupBrokerId); + if (!$connect) { + return false; + } + $topics = \Kafka\ConsumerConfig::getInstance()->getTopics(); + $assign = \Kafka\Consumer\Assignment::getInstance(); + $memberId = $assign->getMemberId(); + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + 'session_timeout' => \Kafka\ConsumerConfig::getInstance()->getSessionTimeout(), + 'rebalance_timeout' => \Kafka\ConsumerConfig::getInstance()->getRebalanceTimeout(), + 'member_id' => ($memberId == null) ? '' : $memberId, + 'data' => array( + array( + 'protocol_name' => 'range', + 'version' => 0, + 'subscription' => $topics, + 'user_data' => '', + ), + ), + ); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::JOIN_GROUP_REQUEST, $params); + $connect->write($requestData); + $this->debug("Join group start, params:" . json_encode($params)); + } + + // }}} + // {{{ public function failJoinGroup() + + public function failJoinGroup($errorCode) + { + $assign = \Kafka\Consumer\Assignment::getInstance(); + $memberId = $assign->getMemberId(); + $error = sprintf('Join group fail, need rejoin, errorCode %d, memberId: %s', $errorCode, $memberId); + $this->error($error); + $this->stateConvert($errorCode); + } + + // }}} + // {{{ public function succJoinGroup() + + public function succJoinGroup($result) + { + $this->state->succRun(\Kafka\Consumer\State::REQUEST_JOINGROUP); + $assign = \Kafka\Consumer\Assignment::getInstance(); + $assign->setMemberId($result['memberId']); + $assign->setGenerationId($result['generationId']); + if ($result['leaderId'] == $result['memberId']) { // leader assign partition + $assigns = $assign->assign($result['members']); + } + $msg = sprintf('Join group sucess, params: %s', json_encode($result)); + $this->debug($msg); + } + + // }}} + // {{{ public function syncGroup() + + public function syncGroup() + { + $broker = \Kafka\Broker::getInstance(); + $groupBrokerId = $broker->getGroupBrokerId(); + $connect = $broker->getMetaConnect($groupBrokerId); + if (!$connect) { + return; + } + $topics = \Kafka\ConsumerConfig::getInstance()->getTopics(); + $assign = \Kafka\Consumer\Assignment::getInstance(); + $memberId = $assign->getMemberId(); + $generationId = $assign->getGenerationId(); + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + 'generation_id' => $generationId, + 'member_id' => $memberId, + 'data' => $assign->getAssignments(), + ); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::SYNC_GROUP_REQUEST, $params); + $this->debug("Sync group start, params:" . json_encode($params)); + $connect->write($requestData); + } + + // }}} + // {{{ public function failSyncGroup() + + public function failSyncGroup($errorCode) + { + $error = sprintf('Sync group fail, need rejoin, errorCode %d', $errorCode); + $this->error($error); + $this->stateConvert($errorCode); + } + + // }}} + // {{{ public function succSyncGroup() + + public function succSyncGroup($result) + { + $msg = sprintf('Sync group sucess, params: %s', json_encode($result)); + $this->debug($msg); + $this->state->succRun(\Kafka\Consumer\State::REQUEST_SYNCGROUP); + + $topics = \Kafka\Broker::getInstance()->getTopics(); + $brokerToTopics = array(); + foreach ($result['partitionAssignments'] as $topic) { + foreach ($topic['partitions'] as $partId) { + $brokerId = $topics[$topic['topicName']][$partId]; + if (!isset($brokerToTopics[$brokerId])) { + $brokerToTopics[$brokerId] = array(); + } + + $topicInfo = array(); + if (isset($brokerToTopics[$brokerId][$topic['topicName']])) { + $topicInfo = $brokerToTopics[$brokerId][$topic['topicName']]; + } + $topicInfo['topic_name'] = $topic['topicName']; + if (!isset($topicInfo['partitions'])) { + $topicInfo['partitions'] = array(); + } + $topicInfo['partitions'][] = $partId; + $brokerToTopics[$brokerId][$topic['topicName']] = $topicInfo; + } + } + $assign = \Kafka\Consumer\Assignment::getInstance(); + $assign->setTopics($brokerToTopics); + } + + // }}} + // {{{ protected function heartbeat() + + protected function heartbeat() + { + $broker = \Kafka\Broker::getInstance(); + $groupBrokerId = $broker->getGroupBrokerId(); + $connect = $broker->getMetaConnect($groupBrokerId); + if (!$connect) { + return; + } + $assign = \Kafka\Consumer\Assignment::getInstance(); + $memberId = $assign->getMemberId(); + if (!$memberId) { + return; + } + $generationId = $assign->getGenerationId(); + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + 'generation_id' => $generationId, + 'member_id' => $memberId, + ); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::HEART_BEAT_REQUEST, $params); + //$this->debug("Heartbeat group start, params:" . json_encode($params)); + $connect->write($requestData); + } + +// }}} + // {{{ public function failHeartbeat() + + public function failHeartbeat($errorCode) + { + $this->error('Heartbeat error, errorCode:' . $errorCode); + $this->stateConvert($errorCode); + } + + // }}} + // {{{ protected function offset() + + protected function offset() + { + $context = array(); + $broker = \Kafka\Broker::getInstance(); + $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); + foreach ($topics as $brokerId => $topicList) { + $connect = $broker->getMetaConnect($brokerId); + if (!$connect) { + return; + } + $data = array(); + foreach ($topicList as $topic) { + $item = array( + 'topic_name' => $topic['topic_name'], + 'partitions' => array(), + ); + foreach ($topic['partitions'] as $partId) { + $item['partitions'][] = array( + 'partition_id' => $partId, + 'offset' => 1, + 'time' => -1, + ); + $data[] = $item; + } + } + $params = array( + 'replica_id' => -1, + 'data' => $data, + ); + $stream = $connect->getSocket(); + //$this->debug("Get current offset start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_REQUEST, $params); + $connect->write($requestData); + $context[] = (int)$stream; + } + + return $context; + } + + // }}} + // {{{ public function succOffset() + + public function succOffset($result, $fd) + { + $msg = sprintf('Get current offset sucess, result: %s', json_encode($result)); + //$this->debug($msg); + + $offsets = \Kafka\Consumer\Assignment::getInstance()->getOffsets(); + $lastOffsets = \Kafka\Consumer\Assignment::getInstance()->getLastOffsets(); + foreach ($result as $topic) { + foreach ($topic['partitions'] as $part) { + if ($part['errorCode'] != 0) { + $this->stateConvert($part['errorCode']); + break 2; + } + + $offsets[$topic['topicName']][$part['partition']] = end($part['offsets']); + $lastOffsets[$topic['topicName']][$part['partition']] = $part['offsets'][0]; + } + } + \Kafka\Consumer\Assignment::getInstance()->setOffsets($offsets); + \Kafka\Consumer\Assignment::getInstance()->setLastOffsets($lastOffsets); + $this->state->succRun(\Kafka\Consumer\State::REQUEST_OFFSET, $fd); + } + + // }}} + // {{{ protected function fetchOffset() + + protected function fetchOffset() + { + $broker = \Kafka\Broker::getInstance(); + $groupBrokerId = $broker->getGroupBrokerId(); + $connect = $broker->getMetaConnect($groupBrokerId); + if (!$connect) { + return; + } + + $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); + $data = array(); + foreach ($topics as $brokerId => $topicList) { + foreach ($topicList as $topic) { + $partitions = array(); + if (isset($data[$topic['topic_name']]['partitions'])) { + $partitions = $data[$topic['topic_name']]['partitions']; + } + foreach ($topic['partitions'] as $partId) { + $partitions[] = $partId; + } + $data[$topic['topic_name']]['partitions'] = $partitions; + $data[$topic['topic_name']]['topic_name'] = $topic['topic_name']; + } + } + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + 'data' => $data, + ); + //$this->debug("Get current fetch offset start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_FETCH_REQUEST, $params); + $connect->write($requestData); + } + + // }}} + // {{{ public function succFetchOffset() + + public function succFetchOffset($result) + { + $msg = sprintf('Get current fetch offset sucess, result: %s', json_encode($result)); + $this->debug($msg); + + $assign = \Kafka\Consumer\Assignment::getInstance(); + $offsets = $assign->getFetchOffsets(); + foreach ($result as $topic) { + foreach ($topic['partitions'] as $part) { + if ($part['errorCode'] != 0) { + $this->stateConvert($part['errorCode']); + break 2; + } + + $offsets[$topic['topicName']][$part['partition']] = $part['offset']; + } + } + $assign->setFetchOffsets($offsets); + + $consumerOffsets = $assign->getConsumerOffsets(); + $lastOffsets = $assign->getLastOffsets(); + if (empty($consumerOffsets)) { + $consumerOffsets = $assign->getFetchOffsets(); + foreach ($consumerOffsets as $topic => $value) { + foreach ($value as $partId => $offset) { + if (isset($lastOffsets[$topic][$partId]) && $lastOffsets[$topic][$partId] > $offset) { + $consumerOffsets[$topic][$partId] = $offset + 1; + } + } + } + $assign->setConsumerOffsets($consumerOffsets); + $assign->setCommitOffsets($assign->getFetchOffsets()); + } + $this->state->succRun(\Kafka\Consumer\State::REQUEST_FETCH_OFFSET); + } + + // }}} + // {{{ protected function fetch() + + protected function fetch() + { + $this->messages = array(); + $context = array(); + $broker = \Kafka\Broker::getInstance(); + $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); + $consumerOffsets = \Kafka\Consumer\Assignment::getInstance()->getConsumerOffsets(); + foreach ($topics as $brokerId => $topicList) { + $connect = $broker->getDataConnect($brokerId); + if (!$connect) { + return; + } + + $data = array(); + foreach ($topicList as $topic) { + $item = array( + 'topic_name' => $topic['topic_name'], + 'partitions' => array(), + ); + foreach ($topic['partitions'] as $partId) { + $item['partitions'][] = array( + 'partition_id' => $partId, + 'offset' => isset($consumerOffsets[$topic['topic_name']][$partId]) ? $consumerOffsets[$topic['topic_name']][$partId] : 0, + 'max_bytes' => \Kafka\ConsumerConfig::getInstance()->getMaxBytes(), + ); + } + $data[] = $item; + } + $params = array( + 'max_wait_time' => \Kafka\ConsumerConfig::getInstance()->getMaxWaitTime(), + 'replica_id' => -1, + 'min_bytes' => '1000', + 'data' => $data, + ); + $this->debug("Fetch message start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::FETCH_REQUEST, $params); + $connect->write($requestData); + $context[] = (int)$connect->getSocket(); + } + return $context; + } + + // }}} + // {{{ public function succFetch() + + public function succFetch($result, $fd) + { + $assign = \Kafka\Consumer\Assignment::getInstance(); + $this->debug('Fetch success, result:' . json_encode($result)); + foreach ($result['topics'] as $topic) { + foreach ($topic['partitions'] as $part) { + $context = array( + $topic['topicName'], + $part['partition'], + ); + if ($part['errorCode'] != 0) { + $this->stateConvert($part['errorCode'], $context); + continue; + } + + $offset = $assign->getConsumerOffset($topic['topicName'], $part['partition']); + if ($offset === false) { + return; // current is rejoin.... + } + foreach ($part['messages'] as $message) { + $this->messages[$topic['topicName']][$part['partition']][] = $message; + //if ($this->consumer != null) { + // call_user_func($this->consumer, $topic['topicName'], $part['partition'], $message); + //} + $offset = $message['offset']; + } + + $consumerOffset = ($part['highwaterMarkOffset'] > $offset) ? ($offset + 1) : $offset; + $assign->setConsumerOffset($topic['topicName'], $part['partition'], $consumerOffset); + $assign->setCommitOffset($topic['topicName'], $part['partition'], $offset); + } + } + $this->state->succRun(\Kafka\Consumer\State::REQUEST_FETCH, $fd); + } + + // }}} + // {{{ protected function commit() + + protected function consume_msg() + { + foreach ($this->messages as $topic => $value) { + foreach ($value as $part => $messages) { + foreach ($messages as $message) { + if ($this->consumer != null) { + call_user_func($this->consumer, $topic, $part, $message); + } + } + } + } + + $this->messages = array(); + } + + + protected function commit() + { + $config= ConsumerConfig::getInstance(); + if($config->getConsumeMode() == ConsumerConfig::CONSUME_BEFORE_COMMIT_OFFSET) + { + $this->consume_msg(); + } + + + $broker = \Kafka\Broker::getInstance(); + $groupBrokerId = $broker->getGroupBrokerId(); + $connect = $broker->getMetaConnect($groupBrokerId); + if (!$connect) { + return; + } + + $commitOffsets = \Kafka\Consumer\Assignment::getInstance()->getCommitOffsets(); + $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); + \Kafka\Consumer\Assignment::getInstance()->setPrecommitOffsets($commitOffsets); + $data = array(); + foreach ($topics as $brokerId => $topicList) { + foreach ($topicList as $topic) { + $partitions = array(); + if (isset($data[$topic['topic_name']]['partitions'])) { + $partitions = $data[$topic['topic_name']]['partitions']; + } + foreach ($topic['partitions'] as $partId) { + if ($commitOffsets[$topic['topic_name']][$partId] == -1) { + continue; + } + $partitions[$partId]['partition'] = $partId; + $partitions[$partId]['offset'] = $commitOffsets[$topic['topic_name']][$partId]; + } + $data[$topic['topic_name']]['partitions'] = $partitions; + $data[$topic['topic_name']]['topic_name'] = $topic['topic_name']; + } + } + $params = array( + 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), + 'generation_id' => \Kafka\Consumer\Assignment::getInstance()->getGenerationId(), + 'member_id' => \Kafka\Consumer\Assignment::getInstance()->getMemberId(), + 'data' => $data, + ); + $this->debug("Commit current fetch offset start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_COMMIT_REQUEST, $params); + $connect->write($requestData); + } + + // }}} + // {{{ public function succCommit() + + /** + * @var State + */ + public $state; + public function succCommit($result) + { + $this->debug('Commit success, result:' . json_encode($result)); + $this->state->succRun(\Kafka\Consumer\State::REQUEST_COMMIT_OFFSET); + foreach ($result as $topic) { + foreach ($topic['partitions'] as $part) { + if ($part['errorCode'] != 0) { + $this->stateConvert($part['errorCode']); + return; // not call user consumer function + } + } + } + if(ConsumerConfig::getInstance()->getConsumeMode() == ConsumerConfig::CONSUME_AFTER_COMMIT_OFFSET) + { + $this->consume_msg(); + } + } + + // }}} + // {{{ protected function stateConvert() + + protected function stateConvert($errorCode, $context = null) + { + $retry = false; + $this->error(\Kafka\Protocol::getError($errorCode)); + $recoverCodes = array( + \Kafka\Protocol::UNKNOWN_TOPIC_OR_PARTITION, + \Kafka\Protocol::NOT_LEADER_FOR_PARTITION, + \Kafka\Protocol::BROKER_NOT_AVAILABLE, + \Kafka\Protocol::GROUP_LOAD_IN_PROGRESS, + \Kafka\Protocol::GROUP_COORDINATOR_NOT_AVAILABLE, + \Kafka\Protocol::NOT_COORDINATOR_FOR_GROUP, + \Kafka\Protocol::INVALID_TOPIC, + \Kafka\Protocol::INCONSISTENT_GROUP_PROTOCOL, + \Kafka\Protocol::INVALID_GROUP_ID, + ); + $rejoinCodes = array( + \Kafka\Protocol::ILLEGAL_GENERATION, + \Kafka\Protocol::INVALID_SESSION_TIMEOUT, + \Kafka\Protocol::REBALANCE_IN_PROGRESS, + \Kafka\Protocol::UNKNOWN_MEMBER_ID, + ); + + $assign = \Kafka\Consumer\Assignment::getInstance(); + if (in_array($errorCode, $recoverCodes)) { + $this->state->recover(); + $assign->clearOffset(); + return false; + } + + if (in_array($errorCode, $rejoinCodes)) { + if ($errorCode == \Kafka\Protocol::UNKNOWN_MEMBER_ID) { + $assign->setMemberId(''); + } + $assign->clearOffset(); + $this->state->rejoin(); + return false; + } + + if (\Kafka\Protocol::OFFSET_OUT_OF_RANGE == $errorCode) { + $resetOffset = \Kafka\ConsumerConfig::getInstance()->getOffsetReset(); + if ($resetOffset == 'latest') { + $offsets = $assign->getLastOffsets(); + } else { + $offsets = $assign->getOffsets(); + } + list($topic, $partId) = $context; + if (isset($offsets[$topic][$partId])) { + $assign->setConsumerOffset($topic, $partId, $offsets[$topic][$partId]); + } + } + return true; + } + + // }}} + // }}} +} diff --git a/Kafka/Consumer/State.php b/Kafka/Consumer/State.php new file mode 100644 index 00000000..6b77dc8c --- /dev/null +++ b/Kafka/Consumer/State.php @@ -0,0 +1,457 @@ + array(), + self::REQUEST_GETGROUP => array(), + self::REQUEST_JOINGROUP => array(), + self::REQUEST_SYNCGROUP => array(), + self::REQUEST_HEARTGROUP => array(), + self::REQUEST_OFFSET => array( + 'interval' => 2000, + ), + self::REQUEST_FETCH => array( + 'interval' => 100, + ), + self::REQUEST_FETCH_OFFSET => array( + 'interval' => 2000, + ), + self::REQUEST_COMMIT_OFFSET => array( + 'norepeat' => true, + ), + ); + + // }}} + // {{{ functions + // {{{ public function init() + + public function init() + { + $this->callStatus = array( + self::REQUEST_METADATA => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_GETGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_JOINGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_SYNCGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_HEARTGROUP => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_COMMIT_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + ); + + // instances clear + + // init requests + $config = \Kafka\ConsumerConfig::getInstance(); + foreach ($this->requests as $request => $option) { + switch ($request) { + case self::REQUEST_METADATA: + $this->requests[$request]['interval'] = $config->getMetadataRefreshIntervalMs(); + break; + default: + $this->requests[$request]['interval'] = 1000; + } + } + } + + // }}} + // {{{ public function start() + + public function start() + { + foreach ($this->requests as $request => $option) { + if (isset($option['norepeat']) && $option['norepeat']) { + continue; + } + $interval = isset($option['interval']) ? $option['interval'] : 200; + \Amp\repeat(function ($watcherId) use ($request, $option) { + if ($this->checkRun($request) && $option['func'] != null) { + $context = call_user_func($option['func']); + $this->processing($request, $context); + } + $this->requests[$request]['watcher'] = $watcherId; + }, $msInterval = $interval); + } + + // start sync metadata + if (isset($this->requests[self::REQUEST_METADATA]['func'])) { + $context = call_user_func($this->requests[self::REQUEST_METADATA]['func']); + $this->processing($request, $context); + } + \Amp\repeat(function ($watcherId) { + $this->report(); + }, $msInterval = 1000); + } + + // }}} + // {{{ public function succRun() + + public function succRun($key, $context = null) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + switch ($key) { + case self::REQUEST_METADATA: + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + if ($context) { // if kafka broker is change + $this->recover(); + } + break; + case self::REQUEST_GETGROUP: + case self::REQUEST_JOINGROUP: + case self::REQUEST_SYNCGROUP: + $this->callStatus[$key]['status'] = (self::STATUS_STOP | self::STATUS_FINISH); + break; + case self::REQUEST_HEARTGROUP: + case self::REQUEST_FETCH_OFFSET: + case self::REQUEST_COMMIT_OFFSET: + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + break; + case self::REQUEST_OFFSET: + if (!isset($this->callStatus[$key]['context'])) { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + break; + } + unset($this->callStatus[$key]['context'][$context]); + $contextStatus = $this->callStatus[$key]['context']; + if (empty($contextStatus)) { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + } + break; + case self::REQUEST_FETCH: + if (!isset($this->callStatus[$key]['context'])) { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + break; + } + unset($this->callStatus[$key]['context'][$context]); + $contextStatus = $this->callStatus[$key]['context']; + if (empty($contextStatus)) { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + call_user_func($this->requests[self::REQUEST_COMMIT_OFFSET]['func']); + } + break; + } + } + + // }}} + // {{{ public function failRun() + + public function failRun($key, $context = null) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + switch ($key) { + case self::REQUEST_METADATA: + $this->callStatus[$key]['status'] = self::STATUS_LOOP; + break; + case self::REQUEST_GETGROUP: + case self::REQUEST_JOINGROUP: + case self::REQUEST_SYNCGROUP: + $this->recover(); + break; + } + } + + // }}} + // {{{ public function setCallback() + + public function setCallback($callbacks) + { + foreach ($callbacks as $request => $callback) { + $this->requests[$request]['func'] = $callback; + } + } + + // }}} + // {{{ public function rejoin() + + public function rejoin() + { + $joinGroupStatus = $this->callStatus[self::REQUEST_JOINGROUP]['status']; + if (($joinGroupStatus & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return; + } + + $this->callStatus = array( + self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], + self::REQUEST_GETGROUP => $this->callStatus[self::REQUEST_GETGROUP], + self::REQUEST_JOINGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_SYNCGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_HEARTGROUP => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_COMMIT_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + ); + } + + // }}} + // {{{ public function recover() + + public function recover() + { + $this->callStatus = array( + self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], + self::REQUEST_GETGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_JOINGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_SYNCGROUP => array( + 'status'=> self::STATUS_START, + ), + self::REQUEST_HEARTGROUP => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_FETCH_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_COMMIT_OFFSET => array( + 'status'=> self::STATUS_LOOP, + ), + ); + } + + // }}} + // {{{ protected function checkRun() + + protected function checkRun($key) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + $status = $this->callStatus[$key]['status']; + switch ($key) { + case self::REQUEST_METADATA: + if ($status & self::STATUS_PROCESS == self::STATUS_PROCESS) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + case self::REQUEST_GETGROUP: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $metaStatus = $this->callStatus[self::REQUEST_METADATA]['status']; + if (($metaStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_START) == self::STATUS_START) { + return true; + } + return false; + case self::REQUEST_JOINGROUP: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $groupStatus = $this->callStatus[self::REQUEST_GETGROUP]['status']; + if (($groupStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_START) == self::STATUS_START) { + return true; + } + return false; + case self::REQUEST_SYNCGROUP: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $joinStatus = $this->callStatus[self::REQUEST_JOINGROUP]['status']; + if (($joinStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_START) == self::STATUS_START) { + return true; + } + return false; + case self::REQUEST_HEARTGROUP: + case self::REQUEST_OFFSET: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $syncStatus = $this->callStatus[self::REQUEST_SYNCGROUP]['status']; + if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + case self::REQUEST_FETCH_OFFSET: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $syncStatus = $this->callStatus[self::REQUEST_SYNCGROUP]['status']; + if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + $offsetStatus = $this->callStatus[self::REQUEST_OFFSET]['status']; + if (($offsetStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + case self::REQUEST_FETCH: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $fetchOffsetStatus = $this->callStatus[self::REQUEST_FETCH_OFFSET]['status']; + if (($fetchOffsetStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + $commitOffsetStatus = $this->callStatus[self::REQUEST_COMMIT_OFFSET]['status']; + if (($commitOffsetStatus & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + } + } + + // }}} + // {{{ protected function processing() + + protected function processing($key, $context) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + // set process start time + $this->callStatus[$key]['time'] = microtime(true); + switch ($key) { + case self::REQUEST_METADATA: + case self::REQUEST_GETGROUP: + case self::REQUEST_JOINGROUP: + case self::REQUEST_SYNCGROUP: + case self::REQUEST_HEARTGROUP: + case self::REQUEST_FETCH_OFFSET: + case self::REQUEST_COMMIT_OFFSET: + $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; + break; + case self::REQUEST_OFFSET: + case self::REQUEST_FETCH: + $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; + $contextStatus = array(); + foreach ($context as $fd) { + $contextStatus[$fd] = self::STATUS_PROCESS; + } + $this->callStatus[$key]['context'] = $contextStatus; + break; + + } + } + + // }}} + // {{{ protected function report() + + protected function report() + { + //var_dump($this->callStatus[self::REQUEST_COMMIT_OFFSET]); + } + + // }}} + // }}} +} diff --git a/Kafka/ConsumerConfig.php b/Kafka/ConsumerConfig.php new file mode 100644 index 00000000..b983f225 --- /dev/null +++ b/Kafka/ConsumerConfig.php @@ -0,0 +1,149 @@ + '', + 'sessionTimeout' => 30000, + 'rebalanceTimeout' => 30000, + 'topics' => array(), + 'offsetReset' => 'latest', // earliest + 'maxBytes' => 65536, // 64kb + 'maxWaitTime' => 100, + ); + + // }}} + // {{{ functions + // {{{ public function getGroupId() + + public function getGroupId() + { + $groupId = trim($this->ietGroupId()); + + if ($groupId == false || $groupId == '') { + throw new \Kafka\Exception\Config('Get group id value is invalid, must set it not empty string'); + } + + return $groupId; + } + + // }}} + // {{{ public function setGroupId() + + public function setGroupId($groupId) + { + $groupId = trim($groupId); + if ($groupId == false || $groupId == '') { + throw new \Kafka\Exception\Config('Set group id value is invalid, must set it not empty string'); + } + static::$options['groupId'] = $groupId; + } + + // }}} + // {{{ public function setSessionTimeout() + + public function setSessionTimeout($sessionTimeout) + { + if (!is_numeric($sessionTimeout) || $sessionTimeout < 1 || $sessionTimeout > 3600000) { + throw new \Kafka\Exception\Config('Set session timeout value is invalid, must set it 1 .. 3600000'); + } + static::$options['sessionTimeout'] = $sessionTimeout; + } + + // }}} + // {{{ public function setRebalanceTimeout() + + public function setRebalanceTimeout($rebalanceTimeout) + { + if (!is_numeric($rebalanceTimeout) || $rebalanceTimeout < 1 || $rebalanceTimeout > 3600000) { + throw new \Kafka\Exception\Config('Set rebalance timeout value is invalid, must set it 1 .. 3600000'); + } + static::$options['rebalanceTimeout'] = $rebalanceTimeout; + } + + // }}} + // {{{ public function setOffsetReset() + + public function setOffsetReset($offsetReset) + { + if (!in_array($offsetReset, array('latest', 'earliest'))) { + throw new \Kafka\Exception\Config('Set offset reset value is invalid, must set it `latest` or `earliest`'); + } + static::$options['offsetReset'] = $offsetReset; + } + + // }}} + // {{{ public function getTopics() + + public function getTopics() + { + $topics = $this->ietTopics(); + + if (empty($topics)) { + throw new \Kafka\Exception\Config('Get consumer topics value is invalid, must set it not empty'); + } + + return $topics; + } + + // }}} + // {{{ public function setTopics() + + public function setTopics($topics) + { + if (!is_array($topics) || empty($topics)) { + throw new \Kafka\Exception\Config('Set consumer topics value is invalid, must set it not empty array'); + } + static::$options['topics'] = $topics; + } + + // }}} + // }}} + + protected $runtime_options = [ + 'consume_mode' => self::CONSUME_AFTER_COMMIT_OFFSET + ]; + const CONSUME_AFTER_COMMIT_OFFSET = 1; + const CONSUME_BEFORE_COMMIT_OFFSET = 2; + + public function setConsumeMode($mode) + { + $this->runtime_options['consume_mode'] = $mode; + } + + public function getConsumeMode() + { + return $this->runtime_options['consume_mode']; + } +} diff --git a/Kafka/Exception.php b/Kafka/Exception.php new file mode 100644 index 00000000..f336f1c3 --- /dev/null +++ b/Kafka/Exception.php @@ -0,0 +1,31 @@ +log(LogLevel::EMERGENCY, $message, $context); + } + + /** + * Action must be taken immediately. + * + * Example: Entire website down, database unavailable, etc. This should + * trigger the SMS alerts and wake you up. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function alert($message, array $context = array()) + { + $this->log(LogLevel::ALERT, $message, $context); + } + + /** + * Critical conditions. + * + * Example: Application component unavailable, unexpected exception. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function critical($message, array $context = array()) + { + $this->log(LogLevel::CRITICAL, $message, $context); + } + + /** + * Runtime errors that do not require immediate action but should typically + * be logged and monitored. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function error($message, array $context = array()) + { + $this->log(LogLevel::ERROR, $message, $context); + } + + /** + * Exceptional occurrences that are not errors. + * + * Example: Use of deprecated APIs, poor use of an API, undesirable things + * that are not necessarily wrong. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function warning($message, array $context = array()) + { + $this->log(LogLevel::WARNING, $message, $context); + } + + /** + * Normal but significant events. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function notice($message, array $context = array()) + { + $this->log(LogLevel::NOTICE, $message, $context); + } + + /** + * Interesting events. + * + * Example: User logs in, SQL logs. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function info($message, array $context = array()) + { + $this->log(LogLevel::INFO, $message, $context); + } + + /** + * Detailed debug information. + * + * @param string $message + * @param array $context + * + * @return void + */ + public function debug($message, array $context = array()) + { + $this->log(LogLevel::DEBUG, $message, $context); + } + + /** + * Logs with an arbitrary level. + * + * @param mixed $level + * @param string $message + * @param array $context + * + * @return void + */ + public function log($level, $message, array $context = array()) + { + if ($this->logger == null) { + $this->logger = new NullLogger(); + } + $this->logger->log($level, $message, $context); + } +} diff --git a/Kafka/Producer.php b/Kafka/Producer.php new file mode 100644 index 00000000..f290bd12 --- /dev/null +++ b/Kafka/Producer.php @@ -0,0 +1,129 @@ +process = new \Kafka\Producer\SyncProcess(); + } else { + $this->process = new \Kafka\Producer\Process($producer); + } + } + + // }}} + // {{{ public function send() + + /** + * start producer + * + * @access public + * @data is data is boolean that is async process, thus it is sync process + * @return void + */ + public function send($data = true) + { + if ($this->logger) { + $this->process->setLogger($this->logger); + } + if (is_bool($data)) { + $this->process->start(); + if ($data) { + \Amp\run(); + } + } else { + return $this->process->send($data); + } + } + + // }}} + // {{{ public function syncMeta() + + /** + * syncMeta producer + * + * @access public + * @return void + */ + public function syncMeta() + { + return $this->process->syncMeta(); + } + + // }}} + // {{{ public function success() + + /** + * producer success + * + * @access public + * @return void + */ + public function success(\Closure $success = null) + { + $this->process->setSuccess($success); + } + + // }}} + // {{{ public function error() + + /** + * producer error + * + * @access public + * @return void + */ + public function error(\Closure $error = null) + { + $this->process->setError($error); + } + + // }}} + // }}} +} diff --git a/Kafka/Producer/Process.php b/Kafka/Producer/Process.php new file mode 100644 index 00000000..2e1f16a7 --- /dev/null +++ b/Kafka/Producer/Process.php @@ -0,0 +1,384 @@ +producer = $producer; + } + + // }}} + // {{{ public function init() + + /** + * start consumer + * + * @access public + * @return void + */ + public function init() + { + // init protocol + $config = \Kafka\ProducerConfig::getInstance(); + \Kafka\Protocol::init($config->getBrokerVersion(), $this->logger); + + // init process request + $broker = \Kafka\Broker::getInstance(); + $broker->setProcess(function ($data, $fd) { + $this->processRequest($data, $fd); + }); + + // init state + $this->state = \Kafka\Producer\State::getInstance(); + if ($this->logger) { + $this->state->setLogger($this->logger); + } + $this->state->setCallback(array( + \Kafka\Producer\State::REQUEST_METADATA => function () { + return $this->syncMeta(); + }, + \Kafka\Producer\State::REQUEST_PRODUCE => function () { + return $this->produce(); + }, + )); + $this->state->init(); + + $topics = $broker->getTopics(); + if (!empty($topics)) { + $this->state->succRun(\Kafka\Producer\State::REQUEST_METADATA); + } + } + + // }}} + // {{{ public function start() + + /** + * start consumer + * + * @access public + * @return void + */ + public function start() + { + $this->init(); + $this->state->start(); + $config = \Kafka\ProducerConfig::getInstance(); + $isAsyn = $config->getIsAsyn(); + if (!$isAsyn) { + \Amp\repeat(function ($watcherId) { + if ($this->error) { + call_user_func($this->error, 1000); + } + \Amp\cancel($watcherId); + \Amp\stop(); + }, $msInterval = $config->getRequestTimeout()); + }; + } + + // }}} + // {{{ public function stop() + + /** + * stop consumer + * + * @access public + * @return void + */ + public function stop() + { + $this->isRunning = false; + } + + // }}} + // {{{ public function setSuccess() + + /** + * set success callback + * + * @access public + * @return void + */ + public function setSuccess($success) + { + $this->success = $success; + } + + // }}} + // {{{ public function setError() + + /** + * set error callback + * + * @access public + * @return void + */ + public function setError($error) + { + $this->error = $error; + } + + // }}} + // {{{ public function syncMeta() + + public function syncMeta() + { + $this->debug('Start sync metadata request'); + $brokerList = explode(',', \Kafka\ProducerConfig::getInstance()->getMetadataBrokerList()); + $brokerHost = array(); + foreach ($brokerList as $key => $val) { + if (trim($val)) { + $brokerHost[] = $val; + } + } + if (count($brokerHost) == 0) { + throw new \Kafka\Exception('Not set config `metadataBrokerList`'); + } + shuffle($brokerHost); + $broker = \Kafka\Broker::getInstance(); + foreach ($brokerHost as $host) { + $socket = $broker->getMetaConnect($host); + if ($socket) { + $params = array(); + $this->debug('Start sync metadata request params:' . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); + $socket->write($requestData); + return; + } + } + throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); + } + + // }}} + // {{{ protected function processRequest() + + /** + * process Request + * + * @access public + * @return void + */ + protected function processRequest($data, $fd) + { + $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); + switch ($correlationId) { + case \Kafka\Protocol::METADATA_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); + if (!isset($result['brokers']) || !isset($result['topics'])) { + $this->error('Get metadata is fail, brokers or topics is null.'); + $this->state->failRun(\Kafka\Producer\State::REQUEST_METADATA); + } else { + $broker = \Kafka\Broker::getInstance(); + $isChange = $broker->setData($result['topics'], $result['brokers']); + $this->state->succRun(\Kafka\Producer\State::REQUEST_METADATA, $isChange); + } + break; + case \Kafka\Protocol::PRODUCE_REQUEST: + $result = \Kafka\Protocol::decode(\Kafka\Protocol::PRODUCE_REQUEST, substr($data, 4)); + $this->succProduce($result, $fd); + break; + default: + $this->error('Error request, correlationId:' . $correlationId); + } + } + + // }}} + // {{{ protected function produce() + + protected function produce() + { + $context = array(); + $broker = \Kafka\Broker::getInstance(); + $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); + $timeout = \Kafka\ProducerConfig::getInstance()->getTimeout(); + + // get send message + // data struct + // topic: + // partId: + // key: + // value: + $data = call_user_func($this->producer); + if (empty($data)) { + return $context; + } + + $sendData = $this->convertMessage($data); + foreach ($sendData as $brokerId => $topicList) { + $connect = $broker->getDataConnect($brokerId); + if (!$connect) { + return; + } + + $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); + $params = array( + 'required_ack' => $requiredAck, + 'timeout' => \Kafka\ProducerConfig::getInstance()->getTimeout(), + 'data' => $topicList, + ); + $this->debug("Send message start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::PRODUCE_REQUEST, $params); + if ($requiredAck == 0) { // If it is 0 the server will not send any response + $this->state->succRun(\Kafka\Producer\State::REQUEST_PRODUCE); + } else { + $connect->write($requestData); + $context[] = (int)$connect->getSocket(); + } + } + + return $context; + } + + // }}} + // {{{ protected function succProduce() + + protected function succProduce($result, $fd) + { + $msg = sprintf('Send message sucess, result: %s', json_encode($result)); + $this->debug($msg); + if ($this->success) { + call_user_func($this->success, $result); + } + $this->state->succRun(\Kafka\Producer\State::REQUEST_PRODUCE, $fd); + } + + // }}} + // {{{ protected function stateConvert() + + protected function stateConvert($errorCode, $context = null) + { + $retry = false; + $this->error(\Kafka\Protocol::getError($errorCode)); + if ($this->error) { + call_user_func($this->error, $errorCode); + } + $recoverCodes = array( + \Kafka\Protocol::UNKNOWN_TOPIC_OR_PARTITION, + \Kafka\Protocol::INVALID_REQUIRED_ACKS, + \Kafka\Protocol::RECORD_LIST_TOO_LARGE, + \Kafka\Protocol::NOT_ENOUGH_REPLICAS_AFTER_APPEND, + \Kafka\Protocol::NOT_ENOUGH_REPLICAS, + \Kafka\Protocol::NOT_LEADER_FOR_PARTITION, + \Kafka\Protocol::BROKER_NOT_AVAILABLE, + \Kafka\Protocol::GROUP_LOAD_IN_PROGRESS, + \Kafka\Protocol::GROUP_COORDINATOR_NOT_AVAILABLE, + \Kafka\Protocol::NOT_COORDINATOR_FOR_GROUP, + \Kafka\Protocol::INVALID_TOPIC, + \Kafka\Protocol::INCONSISTENT_GROUP_PROTOCOL, + \Kafka\Protocol::INVALID_GROUP_ID, + ); + if (in_array($errorCode, $recoverCodes)) { + $this->state->recover(); + return false; + } + return true; + } + + // }}} + // {{{ protected function convertMessage() + + protected function convertMessage($data) + { + $sendData = array(); + $broker = \Kafka\Broker::getInstance(); + $topicInfos = $broker->getTopics(); + foreach ($data as $value) { + if (!isset($value['topic']) || !trim($value['topic'])) { + continue; + } + + if (!isset($topicInfos[$value['topic']])) { + continue; + } + + if (!isset($value['value']) || !trim($value['value'])) { + continue; + } + + if (!isset($value['key'])) { + $value['key'] = ''; + } + + $topicMeta = $topicInfos[$value['topic']]; + $partNums = array_keys($topicMeta); + shuffle($partNums); + $partId = 0; + if (!isset($value['partId']) || !isset($topicMeta[$value['partId']])) { + $partId = $partNums[0]; + } else { + $partId = $value['partId']; + } + + $brokerId = $topicMeta[$partId]; + $topicData = array(); + if (isset($sendData[$brokerId][$value['topic']])) { + $topicData = $sendData[$brokerId][$value['topic']]; + } + + $partition = array(); + if (isset($topicData['partitions'][$partId])) { + $partition = $topicData['partitions'][$partId]; + } + + $partition['partition_id'] = $partId; + if (trim($value['key']) != '') { + $partition['messages'][] = array('value' => $value['value'], 'key' => $value['key']); + } else { + $partition['messages'][] = $value['value']; + } + + $topicData['partitions'][$partId] = $partition; + $topicData['topic_name'] = $value['topic']; + $sendData[$brokerId][$value['topic']] = $topicData; + } + + return $sendData; + } + + // }}} + // }}} +} diff --git a/Kafka/Producer/State.php b/Kafka/Producer/State.php new file mode 100644 index 00000000..5b16ee13 --- /dev/null +++ b/Kafka/Producer/State.php @@ -0,0 +1,275 @@ + array(), + self::REQUEST_PRODUCE => array(), + ); + + // }}} + // {{{ functions + // {{{ public function init() + + public function init() + { + $this->callStatus = array( + self::REQUEST_METADATA => array( + 'status'=> self::STATUS_LOOP, + ), + self::REQUEST_PRODUCE => array( + 'status'=> self::STATUS_LOOP, + ), + ); + + // instances clear + + // init requests + $config = \Kafka\ConsumerConfig::getInstance(); + foreach ($this->requests as $request => $option) { + switch ($request) { + case self::REQUEST_METADATA: + $this->requests[$request]['interval'] = $config->getMetadataRefreshIntervalMs(); + break; + default: + $isAsyn = $config->getIsAsyn(); + if ($isAsyn) { + $this->requests[$request]['interval'] = $config->getProduceInterval(); + } else { + $this->requests[$request]['interval'] = 1; + } + } + } + } + + // }}} + // {{{ public function start() + + public function start() + { + foreach ($this->requests as $request => $option) { + $interval = isset($option['interval']) ? $option['interval'] : 200; + \Amp\repeat(function ($watcherId) use ($request, $option) { + if ($this->checkRun($request) && $option['func'] != null) { + $context = call_user_func($option['func']); + $this->processing($request, $context); + } + $this->requests[$request]['watcher'] = $watcherId; + }, $msInterval = $interval); + } + + // start sync metadata + if (isset($this->requests[self::REQUEST_METADATA]['func']) + && $this->callStatus[self::REQUEST_METADATA]['status'] == self::STATUS_LOOP) { + $context = call_user_func($this->requests[self::REQUEST_METADATA]['func']); + $this->processing($request, $context); + } + \Amp\repeat(function ($watcherId) { + $this->report(); + }, $msInterval = 1000); + } + + // }}} + // {{{ public function succRun() + + public function succRun($key, $context = null) + { + $config = \Kafka\ConsumerConfig::getInstance(); + $isAsyn = $config->getIsAsyn(); + if (!isset($this->callStatus[$key])) { + return false; + } + + switch ($key) { + case self::REQUEST_METADATA: + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + if ($context) { // if kafka broker is change + $this->recover(); + } + break; + case self::REQUEST_PRODUCE: + if ($context == null) { + if (!$isAsyn) { + $this->callStatus[$key]['status'] = self::STATUS_FINISH; + \Amp\stop(); + } else { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + } + break; + } + unset($this->callStatus[$key]['context'][$context]); + $contextStatus = $this->callStatus[$key]['context']; + if (empty($contextStatus)) { + if (!$isAsyn) { + $this->callStatus[$key]['status'] = self::STATUS_FINISH; + \Amp\stop(); + } else { + $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); + } + } + break; + } + } + + // }}} + // {{{ public function failRun() + + public function failRun($key, $context = null) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + switch ($key) { + case self::REQUEST_METADATA: + $this->callStatus[$key]['status'] = self::STATUS_LOOP; + break; + case self::REQUEST_PRODUCE: + $this->recover(); + break; + } + } + + // }}} + // {{{ public function setCallback() + + public function setCallback($callbacks) + { + foreach ($callbacks as $request => $callback) { + $this->requests[$request]['func'] = $callback; + } + } + + // }}} + // {{{ public function recover() + + public function recover() + { + $this->callStatus = array( + self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], + self::REQUEST_PRODUCE => array( + 'status'=> self::STATUS_LOOP, + ), + ); + } + + // }}} + // {{{ protected function checkRun() + + protected function checkRun($key) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + $status = $this->callStatus[$key]['status']; + switch ($key) { + case self::REQUEST_METADATA: + if ($status & self::STATUS_PROCESS == self::STATUS_PROCESS) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + case self::REQUEST_PRODUCE: + if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { + return false; + } + $syncStatus = $this->callStatus[self::REQUEST_METADATA]['status']; + if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { + return false; + } + if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { + return true; + } + return false; + } + } + + // }}} + // {{{ protected function processing() + + protected function processing($key, $context) + { + if (!isset($this->callStatus[$key])) { + return false; + } + + // set process start time + $this->callStatus[$key]['time'] = microtime(true); + switch ($key) { + case self::REQUEST_METADATA: + $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; + break; + case self::REQUEST_PRODUCE: + if (empty($context)) { + break; + } + $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; + $contextStatus = array(); + if (is_array($context)) { + foreach ($context as $fd) { + $contextStatus[$fd] = self::STATUS_PROCESS; + } + $this->callStatus[$key]['context'] = $contextStatus; + } + break; + } + } + + // }}} + // {{{ protected function report() + + protected function report() + { + //var_dump($this->callStatus[self::REQUEST_COMMIT_OFFSET]); + } + + // }}} + // }}} +} diff --git a/Kafka/Producer/SyncProcess.php b/Kafka/Producer/SyncProcess.php new file mode 100644 index 00000000..333a9d7e --- /dev/null +++ b/Kafka/Producer/SyncProcess.php @@ -0,0 +1,200 @@ +getBrokerVersion(), $this->logger); + $this->syncMeta(); + } + + // }}} + // {{{ public function send() + + public function send($data) + { + $broker = \Kafka\Broker::getInstance(); + $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); + $timeout = \Kafka\ProducerConfig::getInstance()->getTimeout(); + + // get send message + // data struct + // topic: + // partId: + // key: + // value: + if (empty($data)) { + return false; + } + + $sendData = $this->convertMessage($data); + $result = array(); + foreach ($sendData as $brokerId => $topicList) { + $connect = $broker->getDataConnect($brokerId, true); + if (!$connect) { + return false; + } + + $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); + $params = array( + 'required_ack' => $requiredAck, + 'timeout' => \Kafka\ProducerConfig::getInstance()->getTimeout(), + 'data' => $topicList, + ); + $this->debug("Send message start, params:" . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::PRODUCE_REQUEST, $params); + $connect->write($requestData); + if ($requiredAck != 0) { // If it is 0 the server will not send any response + $dataLen = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, $connect->read(4)); + $data = $connect->read($dataLen); + $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); + $ret = \Kafka\Protocol::decode(\Kafka\Protocol::PRODUCE_REQUEST, substr($data, 4)); + $result[] = $ret; + } + } + return $result; + } + + // }}} + // {{{ public function syncMeta() + + public function syncMeta() + { + $this->debug('Start sync metadata request'); + $brokerList = explode(',', \Kafka\ProducerConfig::getInstance()->getMetadataBrokerList()); + $brokerHost = array(); + foreach ($brokerList as $key => $val) { + if (trim($val)) { + $brokerHost[] = $val; + } + } + if (count($brokerHost) == 0) { + throw new \Kafka\Exception('Not set config `metadataBrokerList`'); + } + shuffle($brokerHost); + $broker = \Kafka\Broker::getInstance(); + foreach ($brokerHost as $host) { + $socket = $broker->getMetaConnect($host, true); + if ($socket) { + $params = array(); + $this->debug('Start sync metadata request params:' . json_encode($params)); + $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); + $socket->write($requestData); + $dataLen = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, $socket->read(4)); + $data = $socket->read($dataLen); + $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); + $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); + if (!isset($result['brokers']) || !isset($result['topics'])) { + throw new \Kafka\Exception('Get metadata is fail, brokers or topics is null.'); + } else { + $broker = \Kafka\Broker::getInstance(); + $broker->setData($result['topics'], $result['brokers']); + } + return; + } + } + throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); + } + + // }}} + // {{{ protected function convertMessage() + + protected function convertMessage($data) + { + $sendData = array(); + $broker = \Kafka\Broker::getInstance(); + $topicInfos = $broker->getTopics(); + foreach ($data as $value) { + if (!isset($value['topic']) || !trim($value['topic'])) { + continue; + } + + if (!isset($topicInfos[$value['topic']])) { + continue; + } + + if (!isset($value['value']) || !trim($value['value'])) { + continue; + } + + if (!isset($value['key'])) { + $value['key'] = ''; + } + + $topicMeta = $topicInfos[$value['topic']]; + $partNums = array_keys($topicMeta); + shuffle($partNums); + $partId = 0; + if (!isset($value['partId']) || !isset($topicMeta[$value['partId']])) { + $partId = $partNums[0]; + } else { + $partId = $value['partId']; + } + + $brokerId = $topicMeta[$partId]; + $topicData = array(); + if (isset($sendData[$brokerId][$value['topic']])) { + $topicData = $sendData[$brokerId][$value['topic']]; + } + + $partition = array(); + if (isset($topicData['partitions'][$partId])) { + $partition = $topicData['partitions'][$partId]; + } + + $partition['partition_id'] = $partId; + if (trim($value['key']) != '') { + $partition['messages'][] = array('value' => $value['value'], 'key' => $value['key']); + } else { + $partition['messages'][] = $value['value']; + } + + $topicData['partitions'][$partId] = $partition; + $topicData['topic_name'] = $value['topic']; + $sendData[$brokerId][$value['topic']] = $topicData; + } + + return $sendData; + } + + // }}} + // }}} +} diff --git a/Kafka/ProducerConfig.php b/Kafka/ProducerConfig.php new file mode 100644 index 00000000..8ff171bb --- /dev/null +++ b/Kafka/ProducerConfig.php @@ -0,0 +1,102 @@ + 1, + 'timeout' => 5000, + 'isAsyn' => false, + 'requestTimeout' => 6000, + 'produceInterval' => 100, + ); + + // }}} + // {{{ functions + // {{{ public function setRequestTimeout() + + public function setRequestTimeout($requestTimeout) + { + if (!is_numeric($requestTimeout) || $requestTimeout < 1 || $requestTimeout > 900000) { + throw new \Kafka\Exception\Config('Set request timeout value is invalid, must set it 1 .. 900000'); + } + static::$options['requestTimeout'] = $requestTimeout; + } + + // }}} + // {{{ public function setProduceInterval() + + public function setProduceInterval($produceInterval) + { + if (!is_numeric($produceInterval) || $produceInterval < 1 || $produceInterval > 900000) { + throw new \Kafka\Exception\Config('Set produce interval timeout value is invalid, must set it 1 .. 900000'); + } + static::$options['produceInterval'] = $produceInterval; + } + + // }}} + // {{{ public function setTimeout() + + public function setTimeout($timeout) + { + if (!is_numeric($timeout) || $timeout < 1 || $timeout > 900000) { + throw new \Kafka\Exception\Config('Set timeout value is invalid, must set it 1 .. 900000'); + } + static::$options['timeout'] = $timeout; + } + + // }}} + // {{{ public function setRequiredAck() + + public function setRequiredAck($requiredAck) + { + if (!is_numeric($requiredAck) || $requiredAck < -1 || $requiredAck > 1000) { + throw new \Kafka\Exception\Config('Set required ack value is invalid, must set it -1 .. 1000'); + } + static::$options['requiredAck'] = $requiredAck; + } + + // }}} + // {{{ public function setIsAsyn() + + public function setIsAsyn($asyn) + { + if (!is_bool($asyn)) { + throw new \Kafka\Exception\Config('Set isAsyn value is invalid, must set it bool value'); + } + static::$options['isAsyn'] = $asyn; + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol.php b/Kafka/Protocol.php new file mode 100644 index 00000000..9fa5706a --- /dev/null +++ b/Kafka/Protocol.php @@ -0,0 +1,294 @@ + 'Produce', + \Kafka\Protocol\Protocol::FETCH_REQUEST => 'Fetch', + \Kafka\Protocol\Protocol::OFFSET_REQUEST => 'Offset', + \Kafka\Protocol\Protocol::METADATA_REQUEST => 'Metadata', + \Kafka\Protocol\Protocol::OFFSET_COMMIT_REQUEST => 'CommitOffset', + \Kafka\Protocol\Protocol::OFFSET_FETCH_REQUEST => 'FetchOffset', + \Kafka\Protocol\Protocol::GROUP_COORDINATOR_REQUEST => 'GroupCoordinator', + \Kafka\Protocol\Protocol::JOIN_GROUP_REQUEST => 'JoinGroup', + \Kafka\Protocol\Protocol::HEART_BEAT_REQUEST => 'Heartbeat', + \Kafka\Protocol\Protocol::LEAVE_GROUP_REQUEST => 'LeaveGroup', + \Kafka\Protocol\Protocol::SYNC_GROUP_REQUEST => 'SyncGroup', + \Kafka\Protocol\Protocol::DESCRIBE_GROUPS_REQUEST => 'DescribeGroups', + \Kafka\Protocol\Protocol::LIST_GROUPS_REQUEST => 'ListGroup', + ); + + $namespace = '\\Kafka\\Protocol\\'; + foreach ($class as $key => $className) { + $class = $namespace . $className; + self::$objects[$key] = new $class($version); + if ($logger) { + self::$objects[$key]->setLogger($logger); + } + } + } + + // }}} + // {{{ public static function encode() + + /** + * request encode + * + * @param key $appkey + * @param array $payloads + * @access public + * @return string + */ + public static function encode($key, $payloads) + { + if (!isset(self::$objects[$key])) { + throw new \Kafka\Exception('Not support api key, key:' . $key); + } + + return self::$objects[$key]->encode($payloads); + } + + // }}} + // {{{ public static function decode() + + /** + * decode response + * + * @access public + * @return array + */ + public static function decode($key, $data) + { + if (!isset(self::$objects[$key])) { + throw new \Kafka\Exception('Not support api key, key:' . $key); + } + + return self::$objects[$key]->decode($data); + } + + // }}} + // {{{ public static function getError() + + /** + * get error + * + * @param integer $errCode + * @static + * @access public + * @return string + */ + public static function getError($errCode) + { + switch ($errCode) { + case 0: + $error = 'No error--it worked!'; + break; + case -1: + $error = 'An unexpected server error'; + break; + case 1: + $error = 'The requested offset is outside the range of offsets maintained by the server for the given topic/partition.'; + break; + case 2: + $error = 'This indicates that a message contents does not match its CRC'; + break; + case 3: + $error = 'This request is for a topic or partition that does not exist on this broker.'; + break; + case 4: + $error = 'The message has a negative size'; + break; + case 5: + $error = 'This error is thrown if we are in the middle of a leadership election and there is currently no leader for this partition and hence it is unavailable for writes'; + break; + case 6: + $error = 'This error is thrown if the client attempts to send messages to a replica that is not the leader for some partition. It indicates that the clients metadata is out of date.'; + break; + case 7: + $error = 'This error is thrown if the request exceeds the user-specified time limit in the request.'; + break; + case 8: + $error = 'This is not a client facing error and is used only internally by intra-cluster broker communication.'; + break; + case 9: + $error = 'The replica is not available for the requested topic-partition'; + break; + case 10: + $error = 'The server has a configurable maximum message size to avoid unbounded memory allocation. This error is thrown if the client attempt to produce a message larger than this maximum.'; + break; + case 11: + $error = 'Internal error code for broker-to-broker communication.'; + break; + case 12: + $error = 'If you specify a string larger than configured maximum for offset metadata'; + break; + case 13: + $error = 'The server disconnected before a response was received.'; + break; + case 14: + $error = 'The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition).'; + break; + case 15: + $error = 'The broker returns this error code for consumer metadata requests or offset commit requests if the offsets topic has not yet been created.'; + break; + case 16: + $error = 'The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for.'; + break; + case 17: + $error = 'The request attempted to perform an operation on an invalid topic.'; + break; + case 18: + $error = 'The request included message batch larger than the configured segment size on the server.'; + break; + case 19: + $error = 'Messages are rejected since there are fewer in-sync replicas than required.'; + break; + case 20: + $error = 'Messages are written to the log, but to fewer in-sync replicas than required.'; + break; + case 21: + $error = 'Produce request specified an invalid value for required acks.'; + break; + case 22: + $error = 'Specified group generation id is not valid.'; + break; + case 23: + $error = 'The group member\'s supported protocols are incompatible with those of existing members.'; + break; + case 24: + $error = 'The configured groupId is invalid'; + break; + case 25: + $error = 'The coordinator is not aware of this member.'; + break; + case 26: + $error = 'The session timeout is not within the range allowed by the broker (as configured by group.min.session.timeout.ms and group.max.session.timeout.ms).'; + break; + case 27: + $error = 'The group is rebalancing, so a rejoin is needed.'; + break; + case 28: + $error = 'The committing offset data size is not valid'; + break; + case 29: + $error = 'Topic authorization failed.'; + break; + case 30: + $error = 'Group authorization failed.'; + break; + case 31: + $error = 'Cluster authorization failed.'; + break; + case 32: + $error = 'The timestamp of the message is out of acceptable range.'; + break; + case 33: + $error = 'The broker does not support the requested SASL mechanism.'; + break; + case 34: + $error = 'Request is not valid given the current SASL state.'; + break; + case 35: + $error = 'The version of API is not supported.'; + break; + default: + $error = 'Unknown error'; + } + + return $error; + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/CommitOffset.php b/Kafka/Protocol/CommitOffset.php new file mode 100644 index 00000000..fc84a077 --- /dev/null +++ b/Kafka/Protocol/CommitOffset.php @@ -0,0 +1,226 @@ +getApiVersion(self::OFFSET_COMMIT_REQUEST); + + $header = $this->requestHeader('kafka-php', self::OFFSET_COMMIT_REQUEST, self::OFFSET_COMMIT_REQUEST); + + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + if ($version == self::API_VERSION1) { + $data .= self::pack(self::BIT_B32, $payloads['generation_id']); + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + } + if ($version == self::API_VERSION2) { + $data .= self::pack(self::BIT_B32, $payloads['generation_id']); + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + $data .= self::pack(self::BIT_B64, $payloads['retention_time']); + } + + $data .= self::encodeArray($payloads['data'], array($this, 'encodeTopic')); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $version = $this->getApiVersion(self::OFFSET_REQUEST); + $topics = $this->decodeArray(substr($data, $offset), array($this, 'decodeTopic'), $version); + $offset += $topics['length']; + + return $topics['data']; + } + + // }}} + // {{{ protected function encodeTopic() + + /** + * encode commit offset topic array + * + * @param array $values + * @access public + * @return array + */ + protected function encodeTopic($values) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given commit offset data invalid. `topic_name` is undefined.'); + } + if (!isset($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given commit offset data invalid. `partitions` is undefined.'); + } + + $data = self::encodeString($values['topic_name'], self::PACK_INT16); + $data .= self::encodeArray($values['partitions'], array($this, 'encodePartition')); + + return $data; + } + + // }}} + // {{{ protected function encodePartition() + + /** + * encode commit offset partition array + * + * @param array $values + * @access public + * @return array + */ + protected function encodePartition($values) + { + if (!isset($values['partition'])) { + throw new \Kafka\Exception\Protocol('given commit offset data invalid. `partition` is undefined.'); + } + if (!isset($values['offset'])) { + throw new \Kafka\Exception\Protocol('given commit offset data invalid. `offset` is undefined.'); + } + if (!isset($values['metadata'])) { + $values['metadata'] = ''; + } + if (!isset($values['timestamp'])) { + $values['timestamp'] = time() * 1000; + } + $version = $this->getApiVersion(self::OFFSET_COMMIT_REQUEST); + + $data = self::pack(self::BIT_B32, $values['partition']); + $data .= self::pack(self::BIT_B64, $values['offset']); + if ($version == self::API_VERSION1) { + $data .= self::pack(self::BIT_B64, $values['timestamp']); + } + $data .= self::encodeString($values['metadata'], self::PACK_INT16); + + return $data; + } + + // }}} + // {{{ protected function decodeTopic() + + /** + * decode commit offset topic response + * + * @param byte[] $data + * @param string $version + * @access protected + * @return array + */ + protected function decodeTopic($data, $version) + { + $offset = 0; + $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicInfo['length']; + + $partitions = $this->decodeArray(substr($data, $offset), array($this, 'decodePartition'), $version); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicInfo['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // {{{ protected function decodePartition() + + /** + * decode commit offset partition response + * + * @param byte[] $data + * @param string $version + * @access protected + * @return array + */ + protected function decodePartition($data, $version) + { + $offset = 0; + + $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + + + return array( + 'length' => $offset, + 'data' => array( + 'partition' => $partitionId, + 'errorCode' => $errorCode, + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/DescribeGroups.php b/Kafka/Protocol/DescribeGroups.php new file mode 100644 index 00000000..e44814a1 --- /dev/null +++ b/Kafka/Protocol/DescribeGroups.php @@ -0,0 +1,201 @@ +requestHeader('kafka-php', self::DESCRIBE_GROUPS_REQUEST, self::DESCRIBE_GROUPS_REQUEST); + $data = self::encodeArray($payloads, array($this, 'encodeString'), self::PACK_INT16); + + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $groups = $this->decodeArray(substr($data, $offset), array($this, 'describeGroup')); + $offset += $groups['length']; + + return $groups['data']; + } + + // }}} + // {{{ protected function describeGroup() + + /** + * decode describe group response + * + * @access protected + * @return array + */ + protected function describeGroup($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $groupId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $groupId['length']; + $state = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $state['length']; + $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $protocolType['length']; + $protocol = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $protocol['length']; + + $members = $this->decodeArray(substr($data, $offset), array($this, 'describeMember')); + $offset += $members['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'errorCode' => $errorCode, + 'groupId' => $groupId['data'], + 'state' => $state['data'], + 'protocolType' => $protocolType['data'], + 'protocol' => $protocol['data'], + 'members' => $members['data'] + ) + ); + } + + // }}} + // {{{ protected function describeMember() + + /** + * decode describe members response + * + * @access protected + * @return array + */ + protected function describeMember($data) + { + $offset = 0; + $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $memberId['length']; + $clientId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $clientId['length']; + $clientHost = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $clientHost['length']; + $metadata = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $metadata['length']; + $assignment = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $assignment['length']; + + $memberAssignment = $assignment['data']; + $memberAssignmentOffset = 0; + $version = self::unpack(self::BIT_B16_SIGNED, substr($memberAssignment, $memberAssignmentOffset, 2)); + $memberAssignmentOffset += 2; + $partitionAssignments = $this->decodeArray(substr($memberAssignment, $memberAssignmentOffset), + array($this, 'describeResponsePartition')); + $memberAssignmentOffset += $partitionAssignments['length']; + $userData = $this->decodeString(substr($memberAssignment, $memberAssignmentOffset), self::BIT_B32); + + $metaData = $metadata['data']; + $metaOffset = 0; + $version = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2)); + $metaOffset += 2; + $topics = $this->decodeArray(substr($metaData, $metaOffset), array($this, 'decodeString'), self::BIT_B16); + $metaOffset += $topics['length']; + $metaUserData = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32); + + + return array( + 'length' => $offset, + 'data' => array( + 'memberId' => $memberId['data'], + 'clientId' => $clientId['data'], + 'clientHost' => $clientHost['data'], + 'metadata' => array( + 'version' => $version, + 'topics' => $topics['data'], + 'userData' => $metaUserData['data'], + ), + 'assignment' => array( + 'version' => $version, + 'partitions' => $partitionAssignments['data'], + 'userData' => $userData['data'] + ) + ) + ); + } + + // }}} + // {{{ protected function describeResponsePartition() + + /** + * decode describe group partition response + * + * @access protected + * @return array + */ + protected function describeResponsePartition($data) + { + $offset = 0; + $topicName = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicName['length']; + $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicName['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Fetch.php b/Kafka/Protocol/Fetch.php new file mode 100644 index 00000000..259a4764 --- /dev/null +++ b/Kafka/Protocol/Fetch.php @@ -0,0 +1,377 @@ +requestHeader('kafka-php', self::FETCH_REQUEST, self::FETCH_REQUEST); + $data = self::pack(self::BIT_B32, $payloads['replica_id']); + $data .= self::pack(self::BIT_B32, $payloads['max_wait_time']); + $data .= self::pack(self::BIT_B32, $payloads['min_bytes']); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeFetchTopic')); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode fetch response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $version = $this->getApiVersion(self::FETCH_REQUEST); + $throttleTime = 0; + if ($version != self::API_VERSION0) { + $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + } + + $topics = $this->decodeArray(substr($data, $offset), array($this, 'fetchTopic'), $version); + $offset += $topics['length']; + + return array( + 'throttleTime' => $throttleTime, + 'topics' => $topics['data'], + ); + } + + // }}} + // {{{ protected function fetchTopic() + + /** + * decode fetch topic response + * + * @access protected + * @return array + */ + protected function fetchTopic($data, $version) + { + $offset = 0; + $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicInfo['length']; + + $partitions = $this->decodeArray(substr($data, $offset), array($this, 'fetchPartition'), $version); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicInfo['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // {{{ protected function fetchPartition() + + /** + * decode fetch partition response + * + * @access protected + * @return array + */ + protected function fetchPartition($data, $version) + { + $offset = 0; + $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $highwaterMarkOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + + $messageSetSize = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + + if ($offset < strlen($data) && $messageSetSize) { + $messages = $this->decodeMessageSetArray(substr($data, $offset, $messageSetSize), array($this, 'decodeMessageSet'), $messageSetSize); + $offset += $messages['length']; + } + + return array( + 'length' => $offset, + 'data' => array( + 'partition' => $partitionId, + 'errorCode' => $errorCode, + 'highwaterMarkOffset' => $highwaterMarkOffset, + 'messageSetSize' => $messageSetSize, + 'messages' => isset($messages['data']) ? $messages['data'] : array(), + ) + ); + } + + // }}} + // {{{ protected function decodeMessageSetArray() + + /** + * decode message Set + * + * @param array $array + * @param Callable $func + * @param null $options + * @return string + * @access protected + */ + protected function decodeMessageSetArray($data, $func, $messageSetSize = null) + { + $offset = 0; + if (!is_callable($func, false)) { + throw new \Kafka\Exception\Protocol('Decode array failed, given function is not callable.'); + } + + $result = array(); + while ($offset < strlen($data)) { + $value = substr($data, $offset); + if (!is_null($messageSetSize)) { + $ret = call_user_func($func, $value, $messageSetSize); + } else { + $ret = call_user_func($func, $value); + } + + if (!is_array($ret) && $ret === false) { + break; + } + + if (!isset($ret['length']) || !isset($ret['data'])) { + throw new \Kafka\Exception\Protocol('Decode array failed, given function return format is invliad'); + } + if ($ret['length'] == 0) { + continue; + } + + $offset += $ret['length']; + $result[] = $ret['data']; + } + + if ($offset < $messageSetSize) { + $offset = $messageSetSize; + } + + return array('length' => $offset, 'data' => $result); + } + + // }}} + // {{{ protected function decodeMessageSet() + + /** + * decode message set + * N.B., MessageSets are not preceded by an int32 like other array elements + * in the protocol. + * + * @param array $messages + * @param int $compression + * @return string + * @access protected + */ + protected function decodeMessageSet($data) + { + if (strlen($data) <= 12) { + return false; + } + $offset = 0; + $roffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + $messageSize = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $ret = $this->decodeMessage(substr($data, $offset), $messageSize); + if (!is_array($ret) && $ret == false) { + return false; + } + $offset += $ret['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'offset' => $roffset, + 'size' => $messageSize, + 'message' => $ret['data'], + ) + ); + } + + // }}} + // {{{ protected function decodeMessage() + + /** + * decode message + * N.B., MessageSets are not preceded by an int32 like other array elements + * in the protocol. + * + * @param array $messages + * @param int $compression + * @return string + * @access protected + */ + protected function decodeMessage($data, $messageSize) + { + if (strlen($data) < $messageSize || !$messageSize) { + return false; + } + + $offset = 0; + $crc = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $magic = self::unpack(self::BIT_B8, substr($data, $offset, 1)); + $offset += 1; + $attr = self::unpack(self::BIT_B8, substr($data, $offset, 1)); + $offset += 1; + $timestamp = 0; + $backOffset = $offset; + try { // try unpack message format v0 and v1, if use v1 unpack fail, try unpack v0 + $version = $this->getApiVersion(self::FETCH_REQUEST); + if ($version == self::API_VERSION2) { + $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + } + + $keyRet = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $keyRet['length']; + + $valueRet = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $valueRet['length']; + if ($offset != $messageSize) { + throw new \Kafka\Exception('pack message fail, message len:' . $messageSize . ' , data unpack offset :' . $offset); + } + } catch (\Kafka\Exception $e) { // try unpack message format v0 + $offset = $backOffset; + $timestamp = 0; + $keyRet = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $keyRet['length']; + + $valueRet = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $valueRet['length']; + } + + return array( + 'length' => $offset, + 'data' => array( + 'crc' => $crc, + 'magic' => $magic, + 'attr' => $attr, + 'timestamp' => $timestamp, + 'key' => $keyRet['data'], + 'value' => $valueRet['data'], + ) + ); + } + + // }}} + // {{{ protected function encodeFetchPartion() + + /** + * encode signal part + * + * @param partions + * @access protected + * @return string + */ + protected function encodeFetchPartion($values) + { + if (!isset($values['partition_id'])) { + throw new \Kafka\Exception\Protocol('given fetch data invalid. `partition_id` is undefined.'); + } + + if (!isset($values['offset'])) { + $values['offset'] = 0; + } + + if (!isset($values['max_bytes'])) { + $values['max_bytes'] = 2 * 1024 * 1024; + } + + $data = self::pack(self::BIT_B32, $values['partition_id']); + $data .= self::pack(self::BIT_B64, $values['offset']); + $data .= self::pack(self::BIT_B32, $values['max_bytes']); + + return $data; + } + + // }}} + // {{{ protected function encodeFetchTopic() + + /** + * encode signal topic + * + * @param partions + * @access protected + * @return string + */ + protected function encodeFetchTopic($values) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given fetch data invalid. `topic_name` is undefined.'); + } + + if (!isset($values['partitions']) || empty($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given fetch data invalid. `partitions` is undefined.'); + } + + $topic = self::encodeString($values['topic_name'], self::PACK_INT16); + $partitions = self::encodeArray($values['partitions'], array($this, 'encodeFetchPartion')); + + return $topic . $partitions; + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/FetchOffset.php b/Kafka/Protocol/FetchOffset.php new file mode 100644 index 00000000..a0274d61 --- /dev/null +++ b/Kafka/Protocol/FetchOffset.php @@ -0,0 +1,185 @@ +requestHeader('kafka-php', self::OFFSET_FETCH_REQUEST, self::OFFSET_FETCH_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeOffsetTopic')); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @param byte[] $data + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $version = $this->getApiVersion(self::OFFSET_REQUEST); + $topics = $this->decodeArray(substr($data, $offset), array($this, 'offsetTopic'), $version); + $offset += $topics['length']; + + return $topics['data']; + } + + // }}} + // {{{ protected function encodeOffsetPartion() + + /** + * encode signal part + * + * @param byte[] values partions data + * @access protected + * @return string + */ + protected function encodeOffsetPartion($values) + { + return self::pack(self::BIT_B32, $values); + } + + // }}} + // {{{ protected function encodeOffsetTopic() + + /** + * encode signal topic + * + * @param partions + * @access protected + * @return string + */ + protected function encodeOffsetTopic($values) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given fetch offset data invalid. `topic_name` is undefined.'); + } + + if (!isset($values['partitions']) || empty($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given fetch offset data invalid. `partitions` is undefined.'); + } + + $topic = self::encodeString($values['topic_name'], self::PACK_INT16); + $partitions = self::encodeArray($values['partitions'], array($this, 'encodeOffsetPartion')); + + return $topic . $partitions; + } + + // }}} + // {{{ protected function offsetTopic() + + /** + * decode offset topic response + * + * @access protected + * @return array + */ + protected function offsetTopic($data, $version) + { + $offset = 0; + $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicInfo['length']; + + $partitions = $this->decodeArray(substr($data, $offset), array($this, 'offsetPartition'), $version); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicInfo['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // {{{ protected function offsetPartition() + + /** + * decode offset partition response + * + * @access protected + * @return array + */ + protected function offsetPartition($data, $version) + { + $offset = 0; + + $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + + $roffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + + $metadata = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $metadata['length']; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + + + return array( + 'length' => $offset, + 'data' => array( + 'partition' => $partitionId, + 'errorCode' => $errorCode, + 'metadata' => $metadata['data'], + 'offset' => $roffset, + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/GroupCoordinator.php b/Kafka/Protocol/GroupCoordinator.php new file mode 100644 index 00000000..13fbd701 --- /dev/null +++ b/Kafka/Protocol/GroupCoordinator.php @@ -0,0 +1,85 @@ +requestHeader('kafka-php', self::GROUP_COORDINATOR_REQUEST, self::GROUP_COORDINATOR_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $coordinatorId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $hosts = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $hosts['length']; + $coordinatorPort = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + + return array( + 'errorCode' => $errorCode, + 'coordinatorId' => $coordinatorId, + 'coordinatorHost' => $hosts['data'], + 'coordinatorPort' => $coordinatorPort + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Heartbeat.php b/Kafka/Protocol/Heartbeat.php new file mode 100644 index 00000000..f7f84654 --- /dev/null +++ b/Kafka/Protocol/Heartbeat.php @@ -0,0 +1,85 @@ +requestHeader('kafka-php', self::HEART_BEAT_REQUEST, self::HEART_BEAT_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data .= self::pack(self::BIT_B32, $payloads['generation_id']); + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode heart beat response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + + return array( + 'errorCode' => $errorCode, + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/JoinGroup.php b/Kafka/Protocol/JoinGroup.php new file mode 100644 index 00000000..0125bd08 --- /dev/null +++ b/Kafka/Protocol/JoinGroup.php @@ -0,0 +1,204 @@ +requestHeader('kafka-php', self::JOIN_GROUP_REQUEST, self::JOIN_GROUP_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data .= self::pack(self::BIT_B32, $payloads['session_timeout']); + if ($this->getApiVersion(self::JOIN_GROUP_REQUEST) == self::API_VERSION1) { + $data .= self::pack(self::BIT_B32, $payloads['rebalance_timeout']); + } + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + $data .= self::encodeString($payloads['protocol_type'], self::PACK_INT16); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeGroupProtocol')); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode join group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $generationId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $groupProtocol = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $groupProtocol['length']; + $leaderId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $leaderId['length']; + $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $memberId['length']; + + $members = $this->decodeArray(substr($data, $offset), array($this, 'joinGroupMember')); + $offset += $memberId['length']; + + return array( + 'errorCode' => $errorCode, + 'generationId' => $generationId, + 'groupProtocol' => $groupProtocol['data'], + 'leaderId' => $leaderId['data'], + 'memberId' => $memberId['data'], + 'members' => $members['data'], + ); + } + + // }}} + // {{{ protected function encodeGroupProtocol() + + /** + * encode group protocol + * + * @param partions + * @access protected + * @return string + */ + protected function encodeGroupProtocol($values) + { + if (!isset($values['protocol_name'])) { + throw new \Kafka\Exception\Protocol('given join group data invalid. `protocol_name` is undefined.'); + } + + $protocolName = self::encodeString($values['protocol_name'], self::PACK_INT16); + + if (!isset($values['version'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `version` is undefined.'); + } + + if (!isset($values['subscription']) || empty($values['subscription'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `subscription` is undefined.'); + } + if (!isset($values['user_data'])) { + $values['user_data'] = ''; + } + + $data = self::pack(self::BIT_B16, 0); + $data .= self::encodeArray($values['subscription'], array($this, 'encodeGroupProtocolMetaTopic')); + $data .= self::encodeString($values['user_data'], self::PACK_INT32); + + return $protocolName . self::encodeString($data, self::PACK_INT32); + } + + // }}} + // {{{ protected function encodeGroupProtocolMetaTopic() + + /** + * encode group protocol metadata topic + * + * @param partions + * @access protected + * @return string + */ + protected function encodeGroupProtocolMetaTopic($values) + { + $topic = self::encodeString($values, self::PACK_INT16); + return $topic; + } + + // }}} + // {{{ protected function joinGroupMember() + + /** + * decode join group member response + * + * @access protected + * @return array + */ + protected function joinGroupMember($data) + { + $offset = 0; + $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $memberId['length']; + $memberMeta = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $memberMeta['length']; + + $metaData = $memberMeta['data']; + $metaOffset = 0; + $version = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2)); + $metaOffset += 2; + $topics = $this->decodeArray(substr($metaData, $metaOffset), array($this, 'decodeString'), self::BIT_B16); + $metaOffset += $topics['length']; + $userData = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32); + + return array( + 'length' => $offset, + 'data' => array( + 'memberId' => $memberId['data'], + 'memberMeta' => array( + 'version' => $version, + 'topics' => $topics['data'], + 'userData' => $userData['data'], + ), + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/LeaveGroup.php b/Kafka/Protocol/LeaveGroup.php new file mode 100644 index 00000000..c6add5cf --- /dev/null +++ b/Kafka/Protocol/LeaveGroup.php @@ -0,0 +1,79 @@ +requestHeader('kafka-php', self::LEAVE_GROUP_REQUEST, self::LEAVE_GROUP_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, 0, 2)); + + return array( + 'errorCode' => $errorCode, + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/ListGroup.php b/Kafka/Protocol/ListGroup.php new file mode 100644 index 00000000..6f710375 --- /dev/null +++ b/Kafka/Protocol/ListGroup.php @@ -0,0 +1,99 @@ +requestHeader('kafka-php', self::LIST_GROUPS_REQUEST, self::LIST_GROUPS_REQUEST); + $data = self::encodeString($header, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $groups = $this->decodeArray(substr($data, $offset), array($this, 'listGroup')); + + return array( + 'errorCode' => $errorCode, + 'groups' => $groups['data'], + ); + } + + // }}} + // {{{ protected function listGroup() + + /** + * decode list group response + * + * @access protected + * @return array + */ + protected function listGroup($data) + { + $offset = 0; + $groupId = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $groupId['length']; + $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $protocolType['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'groupId' => $groupId['data'], + 'protocolType' => $protocolType['data'], + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Metadata.php b/Kafka/Protocol/Metadata.php new file mode 100644 index 00000000..37fde7e9 --- /dev/null +++ b/Kafka/Protocol/Metadata.php @@ -0,0 +1,179 @@ +requestHeader('kafka-php', self::METADATA_REQUEST, self::METADATA_REQUEST); + $data = self::encodeArray($topics, array($this, 'encodeString'), self::PACK_INT16); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode metadata response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $version = $this->getApiVersion(self::METADATA_REQUEST); + $brokerRet = $this->decodeArray(substr($data, $offset), array($this, 'metaBroker'), $version); + $offset += $brokerRet['length']; + $topicMetaRet = $this->decodeArray(substr($data, $offset), array($this, 'metaTopicMetaData'), $version); + $offset += $topicMetaRet['length']; + + $result = array( + 'brokers' => $brokerRet['data'], + 'topics' => $topicMetaRet['data'], + ); + return $result; + } + + // }}} + // {{{ protected function metaBroker() + + /** + * decode meta broker response + * + * @access protected + * @return array + */ + protected function metaBroker($data, $version) + { + $offset = 0; + $nodeId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $hostNameInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $hostNameInfo['length']; + $port = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + return array( + 'length' => $offset, + 'data' => array( + 'host' => $hostNameInfo['data'], + 'port' => $port, + 'nodeId' => $nodeId + ) + ); + } + + // }}} + // {{{ protected function metaTopicMetaData() + + /** + * decode meta topic meta data response + * + * @access protected + * @return array + */ + protected function metaTopicMetaData($data, $version) + { + $offset = 0; + $topicErrCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicInfo['length']; + $partionsMetaRet = $this->decodeArray(substr($data, $offset), array($this, 'metaPartitionMetaData'), $version); + $offset += $partionsMetaRet['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicInfo['data'], + 'errorCode' => $topicErrCode, + 'partitions' => $partionsMetaRet['data'], + ) + ); + } + + // }}} + // {{{ protected function metaPartitionMetaData() + + /** + * decode meta partition meta data response + * + * @access protected + * @return array + */ + protected function metaPartitionMetaData($data, $version) + { + $offset = 0; + $errcode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $partId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $leader = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $replicas = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); + $offset += $replicas['length']; + $isr = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); + $offset += $isr['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'partitionId' => $partId, + 'errorCode' => $errcode, + 'replicas' => $replicas['data'], + 'leader' => $leader, + 'isr' => $isr['data'], + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Offset.php b/Kafka/Protocol/Offset.php new file mode 100644 index 00000000..53cb405a --- /dev/null +++ b/Kafka/Protocol/Offset.php @@ -0,0 +1,203 @@ +requestHeader('kafka-php', self::OFFSET_REQUEST, self::OFFSET_REQUEST); + $data = self::pack(self::BIT_B32, $payloads['replica_id']); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeOffsetTopic')); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + + $version = $this->getApiVersion(self::OFFSET_REQUEST); + $topics = $this->decodeArray(substr($data, $offset), array($this, 'offsetTopic'), $version); + $offset += $topics['length']; + + return $topics['data']; + } + + // }}} + // {{{ protected function encodeOffsetPartion() + + /** + * encode signal part + * + * @param partions + * @access protected + * @return string + */ + protected function encodeOffsetPartion($values) + { + if (!isset($values['partition_id'])) { + throw new \Kafka\Exception\Protocol('given offset data invalid. `partition_id` is undefined.'); + } + + if (!isset($values['time'])) { + $values['time'] = -1; // -1 + } + + if (!isset($values['max_offset'])) { + $values['max_offset'] = 100000; + } + + $data = self::pack(self::BIT_B32, $values['partition_id']); + $data .= self::pack(self::BIT_B64, $values['time']); + + if ($this->getApiVersion(self::OFFSET_REQUEST) == self::API_VERSION0) { + $data .= self::pack(self::BIT_B32, $values['max_offset']); + } + + return $data; + } + + // }}} + // {{{ protected function encodeOffsetTopic() + + /** + * encode signal topic + * + * @param partions + * @access protected + * @return string + */ + protected function encodeOffsetTopic($values) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given offset data invalid. `topic_name` is undefined.'); + } + + if (!isset($values['partitions']) || empty($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given offset data invalid. `partitions` is undefined.'); + } + + $topic = self::encodeString($values['topic_name'], self::PACK_INT16); + $partitions = self::encodeArray($values['partitions'], array($this, 'encodeOffsetPartion')); + + return $topic . $partitions; + } + + // }}} + // {{{ protected function offsetTopic() + + /** + * decode offset topic response + * + * @access protected + * @return array + */ + protected function offsetTopic($data, $version) + { + $offset = 0; + $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicInfo['length']; + + $partitions = $this->decodeArray(substr($data, $offset), array($this, 'offsetPartition'), $version); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicInfo['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // {{{ protected function offsetPartition() + + /** + * decode offset partition response + * + * @access protected + * @return array + */ + protected function offsetPartition($data, $version) + { + $offset = 0; + $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $timestamp = 0; + if ($version != self::API_VERSION0) { + $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + } + $offsets = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B64); + $offset += $offsets['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'partition' => $partitionId, + 'errorCode' => $errorCode, + 'timestamp' => $timestamp, + 'offsets' => $offsets['data'], + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Produce.php b/Kafka/Protocol/Produce.php new file mode 100644 index 00000000..55ca6579 --- /dev/null +++ b/Kafka/Protocol/Produce.php @@ -0,0 +1,276 @@ +requestHeader('kafka-php', 0, self::PRODUCE_REQUEST); + $data = self::pack(self::BIT_B16, $payloads['required_ack']); + $data .= self::pack(self::BIT_B32, $payloads['timeout']); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeProcudeTopic'), self::COMPRESSION_NONE); + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode produce response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $version = $this->getApiVersion(self::PRODUCE_REQUEST); + $ret = $this->decodeArray(substr($data, $offset), array($this, 'produceTopicPair'), $version); + $offset += $ret['length']; + $throttleTime = 0; + if ($version == self::API_VERSION2) { + $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + } + return array('throttleTime' => $throttleTime, 'data' => $ret['data']); + } + + // }}} + // {{{ protected function encodeMessageSet() + + /** + * encode message set + * N.B., MessageSets are not preceded by an int32 like other array elements + * in the protocol. + * + * @param array $messages + * @param int $compression + * @return string + * @static + * @access public + */ + protected function encodeMessageSet($messages, $compression = self::COMPRESSION_NONE) + { + if (!is_array($messages)) { + $messages = array($messages); + } + + $data = ''; + $next = 0; + foreach ($messages as $message) { + $tmpMessage = $this->encodeMessage($message, $compression); + + // int64 -- message offset Message + //This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. (see more details about compressed messages in Kafka below) + $data .= self::pack(self::BIT_B64, $next) . self::encodeString($tmpMessage, self::PACK_INT32); + $next++; + } + return $data; + } + + // }}} + // {{{ protected function encodeMessage() + + /** + * encode signal message + * + * @param string $message + * @param int $compression + * @return string + * @static + * @access protected + */ + protected function encodeMessage($message, $compression = self::COMPRESSION_NONE) + { + // int8 -- magic int8 -- attribute + $version = $this->getApiVersion(self::PRODUCE_REQUEST); + $magic = ($version == self::API_VERSION2) ? self::MESSAGE_MAGIC_VERSION0 : self::MESSAGE_MAGIC_VERSION1; + $data = self::pack(self::BIT_B8, $magic); + $data .= self::pack(self::BIT_B8, $compression); + + $key = ''; + if (is_array($message)) { + $key = $message['key']; + $message = $message['value']; + } + // message key + $data .= self::encodeString($key, self::PACK_INT32); + + // message value + $data .= self::encodeString($message, self::PACK_INT32, $compression); + + $crc = crc32($data); + + // int32 -- crc code string data + $message = self::pack(self::BIT_B32, $crc) . $data; + + return $message; + } + + // }}} + // {{{ protected function encodeProcudePartion() + + /** + * encode signal part + * + * @param $values + * @param $compression + * @return string + * @internal param $partions + * @access protected + */ + protected function encodeProcudePartion($values, $compression) + { + if (!isset($values['partition_id'])) { + throw new \Kafka\Exception\Protocol('given produce data invalid. `partition_id` is undefined.'); + } + + if (!isset($values['messages']) || empty($values['messages'])) { + throw new \Kafka\Exception\Protocol('given produce data invalid. `messages` is undefined.'); + } + + $data = self::pack(self::BIT_B32, $values['partition_id']); + $data .= self::encodeString($this->encodeMessageSet($values['messages'], $compression), self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ protected function encodeProcudeTopic() + + /** + * encode signal topic + * + * @param $values + * @param $compression + * @return string + * @internal param $partions + * @access protected + */ + protected function encodeProcudeTopic($values, $compression) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given produce data invalid. `topic_name` is undefined.'); + } + + if (!isset($values['partitions']) || empty($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given produce data invalid. `partitions` is undefined.'); + } + + $topic = self::encodeString($values['topic_name'], self::PACK_INT16); + $partitions = self::encodeArray($values['partitions'], array($this, 'encodeProcudePartion'), $compression); + + return $topic . $partitions; + } + + // }}} + // {{{ protected function produceTopicPair() + + /** + * decode produce topic pair response + * + * @access protected + * @return array + */ + protected function produceTopicPair($data, $version) + { + $offset = 0; + $topicInfo = $this->decodeString($data, self::BIT_B16); + $offset += $topicInfo['length']; + $ret = $this->decodeArray(substr($data, $offset), array($this, 'producePartitionPair'), $version); + $offset += $ret['length']; + + return array('length' => $offset, 'data' => array( + 'topicName' => $topicInfo['data'], + 'partitions'=> $ret['data'], + )); + } + + // }}} + // {{{ protected function producePartitionPair() + + /** + * decode produce partition pair response + * + * @access protected + * @return array + */ + protected function producePartitionPair($data, $version) + { + $offset = 0; + $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + $partitionOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + $timestamp = 0; + if ($version == self::API_VERSION2) { + $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + } + + return array( + 'length' => $offset, + 'data' => array( + 'partition' => $partitionId, + 'errorCode' => $errorCode, + 'offset' => $offset, + 'timestamp' => $timestamp, + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/Protocol.php b/Kafka/Protocol/Protocol.php new file mode 100644 index 00000000..cc6d74b1 --- /dev/null +++ b/Kafka/Protocol/Protocol.php @@ -0,0 +1,679 @@ +version = $version; + } + + // }}} + // {{{ public static function Khex2bin() + + /** + * hex to bin + * + * @param string $string + * @static + * @access protected + * @return string (raw) + */ + public static function Khex2bin($string) + { + if (function_exists('\hex2bin')) { + return \hex2bin($string); + } else { + $bin = ''; + $len = strlen($string); + for ($i = 0; $i < $len; $i += 2) { + $bin .= pack('H*', substr($string, $i, 2)); + } + + return $bin; + } + } + + // }}} + // {{{ public static function unpack() + + /** + * Unpack a bit integer as big endian long + * + * @static + * @access public + * @param $type + * @param $bytes + * @return int + */ + public static function unpack($type, $bytes) + { + $result = array(); + self::checkLen($type, $bytes); + if ($type == self::BIT_B64) { + $set = unpack($type, $bytes); + $result = ($set[1] & 0xFFFFFFFF) << 32 | ($set[2] & 0xFFFFFFFF); + } elseif ($type == self::BIT_B16_SIGNED) { + // According to PHP docs: 's' = signed short (always 16 bit, machine byte order) + // So lets unpack it.. + $set = unpack($type, $bytes); + + // But if our system is little endian + if (self::isSystemLittleEndian()) { + // We need to flip the endianess because coming from kafka it is big endian + $set = self::convertSignedShortFromLittleEndianToBigEndian($set); + } + $result = $set; + } else { + $result = unpack($type, $bytes); + } + + return is_array($result) ? array_shift($result) : $result; + } + + // }}} + // {{{ public static function pack() + + /** + * pack a bit integer as big endian long + * + * @static + * @access public + * @param $type + * @param $data + * @return int + */ + public static function pack($type, $data) + { + if ($type == self::BIT_B64) { + if ($data == -1) { // -1L + $data = self::Khex2bin('ffffffffffffffff'); + } elseif ($data == -2) { // -2L + $data = self::Khex2bin('fffffffffffffffe'); + } else { + $left = 0xffffffff00000000; + $right = 0x00000000ffffffff; + + $l = ($data & $left) >> 32; + $r = $data & $right; + $data = pack($type, $l, $r); + } + } else { + $data = pack($type, $data); + } + + return $data; + } + + // }}} + // {{{ protected static function checkLen() + + /** + * check unpack bit is valid + * + * @param string $type + * @param string(raw) $bytes + * @static + * @access protected + * @return void + */ + protected static function checkLen($type, $bytes) + { + $len = 0; + switch ($type) { + case self::BIT_B64: + $len = 8; + break; + case self::BIT_B32: + $len = 4; + break; + case self::BIT_B16: + $len = 2; + break; + case self::BIT_B16_SIGNED: + $len = 2; + break; + case self::BIT_B8: + $len = 1; + break; + } + + if (strlen($bytes) != $len) { + throw new \Kafka\Exception\Protocol('unpack failed. string(raw) length is ' . strlen($bytes) . ' , TO ' . $type); + } + } + + // }}} + // {{{ public static function isSystemLittleEndian() + + /** + * Determines if the computer currently running this code is big endian or little endian. + * + * @access public + * @return bool - false if big endian, true if little endian + */ + public static function isSystemLittleEndian() + { + // If we don't know if our system is big endian or not yet... + if (is_null(self::$isLittleEndianSystem)) { + // Lets find out + list($endiantest) = array_values(unpack('L1L', pack('V', 1))); + if ($endiantest != 1) { + // This is a big endian system + self::$isLittleEndianSystem = false; + } else { + // This is a little endian system + self::$isLittleEndianSystem = true; + } + } + + return self::$isLittleEndianSystem; + } + + // }}} + // {{{ public static function convertSignedShortFromLittleEndianToBigEndian() + + /** + * Converts a signed short (16 bits) from little endian to big endian. + * + * @param int[] $bits + * @access public + * @return array + */ + public static function convertSignedShortFromLittleEndianToBigEndian($bits) + { + foreach ($bits as $index => $bit) { + + // get LSB + $lsb = $bit & 0xff; + + // get MSB + $msb = $bit >> 8 & 0xff; + + // swap bytes + $bit = $lsb <<8 | $msb; + + if ($bit >= 32768) { + $bit -= 65536; + } + $bits[$index] = $bit; + } + return $bits; + } + + // }}} + // {{{ public function getApiVersion() + + /** + * Get kafka api version according to specifiy kafka broker version + * + * @param int kafka api key + * @access public + * @return int + */ + public function getApiVersion($apikey) + { + switch ($apikey) { + case self::METADATA_REQUEST: + return self::API_VERSION0; + case self::PRODUCE_REQUEST: + if (version_compare($this->version, '0.10.0') >= 0) { + return self::API_VERSION2; + } elseif (version_compare($this->version, '0.9.0') >= 0) { + return self::API_VERSION1; + } else { + return self::API_VERSION0; + } + case self::FETCH_REQUEST: + if (version_compare($this->version, '0.10.0') >= 0) { + return self::API_VERSION2; + } elseif (version_compare($this->version, '0.9.0') >= 0) { + return self::API_VERSION1; + } else { + return self::API_VERSION0; + } + case self::OFFSET_REQUEST: + // todo + return self::API_VERSION0; + if (version_compare($this->version, '0.10.1.0') >= 0) { + return self::API_VERSION1; + } else { + return self::API_VERSION0; + } + case self::GROUP_COORDINATOR_REQUEST: + return self::API_VERSION0; + case self::OFFSET_COMMIT_REQUEST: + if (version_compare($this->version, '0.9.0') >= 0) { + return self::API_VERSION2; + } elseif (version_compare($this->version, '0.8.2') >= 0) { + return self::API_VERSION1; + } else { + return self::API_VERSION0; // supported in 0.8.1 or later + } + case self::OFFSET_FETCH_REQUEST: + if (version_compare($this->version, '0.8.2') >= 0) { + return self::API_VERSION1; // Offset Fetch Request v1 will fetch offset from Kafka + } else { + return self::API_VERSION0;//Offset Fetch Request v0 will fetch offset from zookeeper + } + case self::JOIN_GROUP_REQUEST: + if (version_compare($this->version, '0.10.1.0') >= 0) { + return self::API_VERSION1; + } else { + return self::API_VERSION0; // supported in 0.9.0.0 and greater + } + case self::SYNC_GROUP_REQUEST: + return self::API_VERSION0; + case self::HEART_BEAT_REQUEST: + return self::API_VERSION0; + case self::LEAVE_GROUP_REQUEST: + return self::API_VERSION0; + case self::LIST_GROUPS_REQUEST: + return self::API_VERSION0; + case self::DESCRIBE_GROUPS_REQUEST: + return self::API_VERSION0; + } + + // default + return self::API_VERSION0; + } + + // }}} + // {{{ public static function getApiText() + + /** + * Get kafka api text + * + * @param int kafka api key + * @access public + * @return string + */ + public static function getApiText($apikey) + { + $apis = array( + self::PRODUCE_REQUEST => 'ProduceRequest', + self::FETCH_REQUEST => 'FetchRequest', + self::OFFSET_REQUEST => 'OffsetRequest', + self::METADATA_REQUEST => 'MetadataRequest', + self::OFFSET_COMMIT_REQUEST => 'OffsetCommitRequest', + self::OFFSET_FETCH_REQUEST => 'OffsetFetchRequest', + self::GROUP_COORDINATOR_REQUEST => 'GroupCoordinatorRequest', + self::JOIN_GROUP_REQUEST => 'JoinGroupRequest', + self::HEART_BEAT_REQUEST => 'HeartbeatRequest', + self::LEAVE_GROUP_REQUEST => 'LeaveGroupRequest', + self::SYNC_GROUP_REQUEST => 'SyncGroupRequest', + self::DESCRIBE_GROUPS_REQUEST => 'DescribeGroupsRequest', + self::LIST_GROUPS_REQUEST => 'ListGroupsRequest', + ); + return $apis[$apikey]; + } + + // }}} + // {{{ public function requestHeader() + + /** + * get request header + * + * @param string $clientId + * @param integer $correlationId + * @param integer $apiKey + * @access public + * @return string + */ + public function requestHeader($clientId, $correlationId, $apiKey) + { + // int16 -- apiKey int16 -- apiVersion int32 correlationId + $binData = self::pack(self::BIT_B16, $apiKey); + $binData .= self::pack(self::BIT_B16, $this->getApiVersion($apiKey)); + $binData .= self::pack(self::BIT_B32, $correlationId); + + // concat client id + $binData .= self::encodeString($clientId, self::PACK_INT16); + $msg = sprintf('ClientId: %s ApiKey: %s ApiVersion: %s', $clientId, self::getApiText($apiKey), $this->getApiVersion($apiKey)); + $this->debug('Start Request ' . $msg); + + return $binData; + } + + // }}} + // {{{ public static function encodeString() + + /** + * encode pack string type + * + * @param string $string + * @param int $bytes self::PACK_INT32: int32 big endian order. self::PACK_INT16: int16 big endian order. + * @param int $compression + * @return string + * @static + * @access public + */ + public static function encodeString($string, $bytes, $compression = self::COMPRESSION_NONE) + { + $packLen = ($bytes == self::PACK_INT32) ? self::BIT_B32 : self::BIT_B16; + switch ($compression) { + case self::COMPRESSION_NONE: + break; + case self::COMPRESSION_GZIP: + $string = \gzencode($string); + break; + case self::COMPRESSION_SNAPPY: + // todo + throw new \Kafka\Exception\NotSupported('SNAPPY compression not yet implemented'); + default: + throw new \Kafka\Exception\NotSupported('Unknown compression flag: ' . $compression); + } + return self::pack($packLen, strlen($string)) . $string; + } + + // }}} + // {{{ public static function encodeArray() + + /** + * encode key array + * + * @param array $array + * @param Callable $func + * @param null $options + * @return string + * @static + * @access public + */ + public static function encodeArray(array $array, $func, $options = null) + { + if (!is_callable($func, false)) { + throw new \Kafka\Exception\Protocol('Encode array failed, given function is not callable.'); + } + + $arrayCount = count($array); + + $body = ''; + foreach ($array as $value) { + if (!is_null($options)) { + $body .= call_user_func($func, $value, $options); + } else { + $body .= call_user_func($func, $value); + } + } + + return self::pack(self::BIT_B32, $arrayCount) . $body; + } + + // }}} + // {{{ public function decodeString() + + /** + * decode unpack string type + * + * @param bytes $data + * @param int $bytes self::BIT_B32: int32 big endian order. self::BIT_B16: int16 big endian order. + * @param int $compression + * @return string + * @access public + */ + public function decodeString($data, $bytes, $compression = self::COMPRESSION_NONE) + { + $offset = ($bytes == self::BIT_B32) ? 4 : 2; + $packLen = self::unpack($bytes, substr($data, 0, $offset)); // int16 topic name length + if ($packLen == 4294967295) { // uint32(4294967295) is int32 (-1) + $packLen = 0; + } + + if ($packLen == 0) { + return array('length' => $offset, 'data' => ''); + } + + $data = substr($data, $offset, $packLen); + $offset += $packLen; + + switch ($compression) { + case self::COMPRESSION_NONE: + break; + case self::COMPRESSION_GZIP: + $data = \gzdecode($data); + break; + case self::COMPRESSION_SNAPPY: + // todo + throw new \Kafka\Exception\NotSupported('SNAPPY compression not yet implemented'); + default: + throw new \Kafka\Exception\NotSupported('Unknown compression flag: ' . $compression); + } + return array('length' => $offset, 'data' => $data); + } + + // }}} + // {{{ public function decodeArray() + + /** + * decode key array + * + * @param array $array + * @param Callable $func + * @param null $options + * @return string + * @access public + */ + public function decodeArray($data, $func, $options = null) + { + $offset = 0; + $arrayCount = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + + if (!is_callable($func, false)) { + throw new \Kafka\Exception\Protocol('Decode array failed, given function is not callable.'); + } + + $result = array(); + for ($i = 0; $i < $arrayCount; $i++) { + $value = substr($data, $offset); + if (!is_null($options)) { + $ret = call_user_func($func, $value, $options); + } else { + $ret = call_user_func($func, $value); + } + + if (!is_array($ret) && $ret === false) { + break; + } + + if (!isset($ret['length']) || !isset($ret['data'])) { + throw new \Kafka\Exception\Protocol('Decode array failed, given function return format is invliad'); + } + if ($ret['length'] == 0) { + continue; + } + + $offset += $ret['length']; + $result[] = $ret['data']; + } + + return array('length' => $offset, 'data' => $result); + } + + // }}} + // {{{ public function decodePrimitiveArray() + + /** + * decode primitive type array + * + * @param bytes[] $data + * @param bites $bites + * @return array + * @access public + */ + public function decodePrimitiveArray($data, $bites) + { + $offset = 0; + $arrayCount = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + if ($arrayCount == 4294967295) { + $arrayCount = 0; + } + + $result = array(); + + for ($i = 0; $i < $arrayCount; $i++) { + if ($bites == self::BIT_B64) { + $result[] = self::unpack(self::BIT_B64, substr($data, $offset, 8)); + $offset += 8; + } elseif ($bites == self::BIT_B32) { + $result[] = self::unpack(self::BIT_B32, substr($data, $offset, 4)); + $offset += 4; + } elseif (in_array($bites, array(self::BIT_B16, self::BIT_B16_SIGNED))) { + $result[] = self::unpack($bites, substr($data, $offset, 2)); + $offset += 2; + } elseif ($bites == self::BIT_B8) { + $result[] = self::unpack($bites, substr($data, $offset, 1)); + $offset += 1; + } + } + + return array('length' => $offset, 'data' => $result); + } + + // }}} + // }}} +} diff --git a/Kafka/Protocol/SyncGroup.php b/Kafka/Protocol/SyncGroup.php new file mode 100644 index 00000000..1993aeff --- /dev/null +++ b/Kafka/Protocol/SyncGroup.php @@ -0,0 +1,212 @@ +requestHeader('kafka-php', self::SYNC_GROUP_REQUEST, self::SYNC_GROUP_REQUEST); + $data = self::encodeString($payloads['group_id'], self::PACK_INT16); + $data .= self::pack(self::BIT_B32, $payloads['generation_id']); + $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); + $data .= self::encodeArray($payloads['data'], array($this, 'encodeGroupAssignment')); + + $data = self::encodeString($header . $data, self::PACK_INT32); + + return $data; + } + + // }}} + // {{{ public function decode() + + /** + * decode group response + * + * @access public + * @return array + */ + public function decode($data) + { + $offset = 0; + $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); + $offset += 2; + + $memberAssignments = $this->decodeString(substr($data, $offset), self::BIT_B32); + $offset += $memberAssignments['length']; + + $memberAssignment = $memberAssignments['data']; + if (strlen($memberAssignment)) { + $memberAssignmentOffset = 0; + $version = self::unpack(self::BIT_B16_SIGNED, substr($memberAssignment, $memberAssignmentOffset, 2)); + $memberAssignmentOffset += 2; + $partitionAssignments = $this->decodeArray(substr($memberAssignment, $memberAssignmentOffset), + array($this, 'syncGroupResponsePartition')); + $memberAssignmentOffset += $partitionAssignments['length']; + $userData = $this->decodeString(substr($memberAssignment, $memberAssignmentOffset), self::BIT_B32); + } else { + return array( + 'errorCode' => $errorCode, + ); + } + + return array( + 'errorCode' => $errorCode, + 'partitionAssignments' => $partitionAssignments['data'], + 'version' => $version, + 'userData' => $userData['data'], + ); + } + + // }}} + // {{{ protected function encodeGroupAssignment() + + /** + * encode group assignment protocol + * + * @param partions + * @access protected + * @return string + */ + protected function encodeGroupAssignment($values) + { + if (!isset($values['version'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `version` is undefined.'); + } + if (!isset($values['member_id'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `member_id` is undefined.'); + } + + if (!isset($values['assignments'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `assignments` is undefined.'); + } + if (!isset($values['user_data'])) { + $values['user_data'] = ''; + } + + $memberId = self::encodeString($values['member_id'], self::PACK_INT16); + + $data = self::pack(self::BIT_B16, 0); + $data .= self::encodeArray($values['assignments'], array($this, 'encodeGroupAssignmentTopic')); + $data .= self::encodeString($values['user_data'], self::PACK_INT32); + + return $memberId . self::encodeString($data, self::PACK_INT32); + } + + // }}} + // {{{ protected function encodeGroupAssignmentTopic() + + /** + * encode group assignment topic protocol + * + * @param partions + * @access protected + * @return string + */ + protected function encodeGroupAssignmentTopic($values) + { + if (!isset($values['topic_name'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `topic_name` is undefined.'); + } + if (!isset($values['partitions'])) { + throw new \Kafka\Exception\Protocol('given data invalid. `partitions` is undefined.'); + } + + $topicName = self::encodeString($values['topic_name'], self::PACK_INT16); + + $partitions = self::encodeArray($values['partitions'], array($this, 'encodeGroupAssignmentTopicPartition')); + + return $topicName . $partitions; + } + + // }}} + // {{{ protected function encodeGroupAssignmentTopicPartition() + + /** + * encode group assignment topic protocol + * + * @param partions + * @access protected + * @return string + */ + protected function encodeGroupAssignmentTopicPartition($values) + { + return self::pack(self::BIT_B32, $values); + } + + // }}} + // {{{ protected function syncGroupResponsePartition() + + /** + * decode sync group partition response + * + * @access protected + * @return array + */ + protected function syncGroupResponsePartition($data) + { + $offset = 0; + $topicName = $this->decodeString(substr($data, $offset), self::BIT_B16); + $offset += $topicName['length']; + $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); + $offset += $partitions['length']; + + return array( + 'length' => $offset, + 'data' => array( + 'topicName' => $topicName['data'], + 'partitions' => $partitions['data'], + ) + ); + } + + // }}} + // }}} +} diff --git a/Kafka/SingletonTrait.php b/Kafka/SingletonTrait.php new file mode 100644 index 00000000..b2dd5ced --- /dev/null +++ b/Kafka/SingletonTrait.php @@ -0,0 +1,76 @@ +host = $host; + $this->port = $port; + $this->setRecvTimeoutSec($recvTimeoutSec); + $this->setRecvTimeoutUsec($recvTimeoutUsec); + $this->setSendTimeoutSec($sendTimeoutSec); + $this->setSendTimeoutUsec($sendTimeoutUsec); + } + + /** + * @param float $sendTimeoutSec + */ + public function setSendTimeoutSec($sendTimeoutSec) + { + $this->sendTimeoutSec = $sendTimeoutSec; + } + + /** + * @param float $sendTimeoutUsec + */ + public function setSendTimeoutUsec($sendTimeoutUsec) + { + $this->sendTimeoutUsec = $sendTimeoutUsec; + } + + /** + * @param float $recvTimeoutSec + */ + public function setRecvTimeoutSec($recvTimeoutSec) + { + $this->recvTimeoutSec = $recvTimeoutSec; + } + + /** + * @param float $recvTimeoutUsec + */ + public function setRecvTimeoutUsec($recvTimeoutUsec) + { + $this->recvTimeoutUsec = $recvTimeoutUsec; + } + + /** + * @param int $number + */ + public function setMaxWriteAttempts($number) + { + $this->maxWriteAttempts = $number; + } + + // }}} + // {{{ public function connect() + + /** + * Connects the socket + * + * @access public + * @return void + */ + public function connect() + { + if (!$this->isSocketDead()) { + return; + } + + if (empty($this->host)) { + throw new \Kafka\Exception('Cannot open null host.'); + } + if ($this->port <= 0) { + throw new \Kafka\Exception('Cannot open without port.'); + } + + $this->stream = @fsockopen( + $this->host, + $this->port, + $errno, + $errstr, + $this->sendTimeoutSec + ($this->sendTimeoutUsec / 1000000) + ); + + if ($this->stream == false) { + $error = 'Could not connect to ' + . $this->host . ':' . $this->port + . ' ('.$errstr.' ['.$errno.'])'; + throw new \Kafka\Exception($error); + } + + stream_set_blocking($this->stream, 0); + stream_set_read_buffer($this->stream, 0); + + $this->readWatcher = \Amp\onReadable($this->stream, function () { + do { + if(!$this->isSocketDead($this->stream)){ + $newData = @fread($this->stream, self::READ_MAX_LEN); + }else{ + $this->reconnect(); + return; + } + if ($newData) { + $this->read($newData); + } + } while ($newData); + }); + + $this->writeWatcher = \Amp\onWritable($this->stream, function () { + $this->write(); + }, array('enable' => false)); // <-- let's initialize the watcher as "disabled" + } + + // }}} + // {{{ public function reconnect() + + /** + * reconnect the socket + * + * @access public + * @return void + */ + public function reconnect() + { + $this->close(); + $this->connect(); + } + + // }}} + // {{{ public function getSocket() + + /** + * get the socket + * + * @access public + * @return void + */ + public function getSocket() + { + return $this->stream; + } + + // }}} + // {{{ public function SetOnReadable() + + public $onReadable; + /** + * set on readable callback function + * + * @access public + * @return void + */ + public function SetOnReadable(\Closure $read) + { + $this->onReadable = $read; + } + + // }}} + // {{{ public function close() + + /** + * close the socket + * + * @access public + * @return void + */ + public function close() + { + \Amp\cancel($this->readWatcher); + \Amp\cancel($this->writeWatcher); + if (is_resource($this->stream)) { + fclose($this->stream); + } + $this->readBuffer = ''; + $this->writeBuffer = ''; + $this->readNeedLength = 0; + } + + /** + * checks if the socket is a valid resource + * + * @access public + * @return boolean + */ + public function isResource() + { + return is_resource($this->stream); + } + + // }}} + // {{{ public function read() + + /** + * Read from the socket at most $len bytes. + * + * This method will not wait for all the requested data, it will return as + * soon as any data is received. + * + * @param integer $len Maximum number of bytes to read. + * @param boolean $verifyExactLength Throw an exception if the number of read bytes is less than $len + * + * @return string Binary data + * @throws \Kafka\Exception\SocketEOF + */ + public function read($data) + { + $this->readBuffer .= $data; + do { + if ($this->readNeedLength == 0) { // response start + if (strlen($this->readBuffer) < 4) { + return; + } + $dataLen = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($this->readBuffer, 0, 4)); + $this->readNeedLength = $dataLen; + $this->readBuffer = substr($this->readBuffer, 4); + } + + if (strlen($this->readBuffer) < $this->readNeedLength) { + return; + } + $data = substr($this->readBuffer, 0, $this->readNeedLength); + + $this->readBuffer = substr($this->readBuffer, $this->readNeedLength); + $this->readNeedLength = 0; + call_user_func($this->onReadable, $data, (int)$this->stream); + } while (strlen($this->readBuffer)); + } + + // }}} + // {{{ public function write() + + /** + * Write to the socket. + * + * @param string $buf The data to write + * + * @return integer + * @throws \Kafka\Exception\SocketEOF + */ + public function write($data = null) + { + if ($data != null) { + $this->writeBuffer .= $data; + } + $bytesToWrite = strlen($this->writeBuffer); + $bytesWritten = @fwrite($this->stream, $this->writeBuffer); + + if ($bytesToWrite === $bytesWritten) { + \Amp\disable($this->writeWatcher); + } elseif ($bytesWritten >= 0) { + \Amp\enable($this->writeWatcher); + } elseif ($this->isSocketDead($this->stream)) { + $this->reconnect(); + } + $this->writeBuffer = substr($this->writeBuffer, $bytesWritten); + } + + // }}} + // {{{ protected function isSocketDead() + + /** + * check the stream is close + * + * @return bool + */ + protected function isSocketDead() + { + return !is_resource($this->stream) || @feof($this->stream); + } + + // }}} + // }}} +} diff --git a/Kafka/SocketSync.php b/Kafka/SocketSync.php new file mode 100644 index 00000000..673f7e52 --- /dev/null +++ b/Kafka/SocketSync.php @@ -0,0 +1,394 @@ +host = $host; + $this->port = $port; + $this->setRecvTimeoutSec($recvTimeoutSec); + $this->setRecvTimeoutUsec($recvTimeoutUsec); + $this->setSendTimeoutSec($sendTimeoutSec); + $this->setSendTimeoutUsec($sendTimeoutUsec); + } + + /** + * @param float $sendTimeoutSec + */ + public function setSendTimeoutSec($sendTimeoutSec) + { + $this->sendTimeoutSec = $sendTimeoutSec; + } + + /** + * @param float $sendTimeoutUsec + */ + public function setSendTimeoutUsec($sendTimeoutUsec) + { + $this->sendTimeoutUsec = $sendTimeoutUsec; + } + + /** + * @param float $recvTimeoutSec + */ + public function setRecvTimeoutSec($recvTimeoutSec) + { + $this->recvTimeoutSec = $recvTimeoutSec; + } + + /** + * @param float $recvTimeoutUsec + */ + public function setRecvTimeoutUsec($recvTimeoutUsec) + { + $this->recvTimeoutUsec = $recvTimeoutUsec; + } + + /** + * @param int $number + */ + public function setMaxWriteAttempts($number) + { + $this->maxWriteAttempts = $number; + } + + // }}} + // {{{ public static function createFromStream() + + /** + * Optional method to set the internal stream handle + * + * @static + * @access public + * @param $stream + * @return Socket + */ + public static function createFromStream($stream) + { + $socket = new self('localhost', 0); + $socket->setStream($stream); + return $socket; + } + + // }}} + // {{{ public function setStream() + + /** + * Optional method to set the internal stream handle + * + * @param mixed $stream + * @access public + * @return void + */ + public function setStream($stream) + { + $this->stream = $stream; + } + + // }}} + // {{{ public function connect() + + /** + * Connects the socket + * + * @access public + * @return void + */ + public function connect() + { + if ($this->stream instanceof Client) { + return; + } + + if (empty($this->host)) { + throw new \Kafka\Exception('Cannot open null host.'); + } + if ($this->port <= 0) { + throw new \Kafka\Exception('Cannot open without port.'); + } + + $this->stream = new Client(SWOOLE_SOCK_TCP); + + if (!$this->stream->connect($this->host, $this->port)) { + $error = 'Could not connect to ' + . $this->host . ':' . $this->port + . ' (' . $this->stream->errMsg . ' [' . $this->stream->errMsg . '])'; + throw new \Kafka\Exception($error); + } + +// stream_set_blocking($this->stream, 0); + } + + // }}} + // {{{ public function close() + + /** + * close the socket + * + * @access public + * @return void + */ + public function close() + { + if ($this->stream instanceof Client) { + $this->stream->close(); + } + } + + /** + * checks if the socket is a valid resource + * + * @access public + * @return boolean + */ + public function isResource() + { + return $this->stream instanceof Client && $this->stream->connected; + } + + // }}} + // {{{ public function read() + + /** + * Read from the socket at most $len bytes. + * + * This method will not wait for all the requested data, it will return as + * soon as any data is received. + * + * @param integer $len Maximum number of bytes to read. + * @param boolean $verifyExactLength Throw an exception if the number of read bytes is less than $len + * + * @return string Binary data + * @throws \Kafka\Exception + */ + public function read($len, $verifyExactLength = false) + { + if ($len > self::READ_MAX_LEN) { + throw new \Kafka\Exception('Could not read ' . $len . ' bytes from stream, length too longer.'); + } + + $null = null; + $read = array($this->stream); +// $readable = @stream_select($read, $null, $null, $this->recvTimeoutSec, $this->recvTimeoutUsec); +// if ($readable > 0) { + $remainingBytes = $len; + $data = $chunk = ''; + while ($remainingBytes > 0) { + $chunk = $this->stream->recv(); + if ($chunk === false) { + $this->close(); + throw new \Kafka\Exception('Could not read ' . $len . ' bytes from stream (no data)'); + } + if (strlen($chunk) === 0) { +// // Zero bytes because of EOF? +// if (feof($this->stream)) { +// $this->close(); +// throw new \Kafka\Exception('Unexpected EOF while reading ' . $len . ' bytes from stream (no data)'); +// } +// // Otherwise wait for bytes +// $readable = @stream_select($read, $null, $null, $this->recvTimeoutSec, $this->recvTimeoutUsec); +// if ($readable !== 1) { +// throw new \Kafka\Exception('Timed out reading socket while reading ' . $len . ' bytes with ' . $remainingBytes . ' bytes to go'); +// } + continue; // attempt another read + } + $data .= $chunk; + $remainingBytes -= strlen($chunk); + } + if ($len === $remainingBytes || ($verifyExactLength && $len !== strlen($data))) { + // couldn't read anything at all OR reached EOF sooner than expected + $this->close(); + throw new \Kafka\Exception('Read ' . strlen($data) . ' bytes instead of the requested ' . $len . ' bytes'); + } + + return $data; +// } +// if (false !== $readable) { +// $res = stream_get_meta_data($this->stream); +// if (!empty($res['timed_out'])) { +// $this->close(); +// throw new \Kafka\Exception('Timed out reading ' . $len . ' bytes from stream'); +// } +// } +// $this->close(); +// throw new \Kafka\Exception('Could not read ' . $len . ' bytes from stream (not readable)'); + } + + // }}} + // {{{ public function write() + + /** + * Write to the socket. + * + * @param string $buf The data to write + * + * @return integer + * @throws Exception + */ + public function write(string $buf) + { + $null = null; + + // fwrite to a socket may be partial, so loop until we + // are done with the entire buffer + $failedWriteAttempts = 0; + $written = 0; + $buflen = strlen($buf); + while ($written < $buflen) { + if ($buflen - $written > self::MAX_WRITE_BUFFER) { + // write max buffer size + $wrote = $this->stream->send(substr($buf, $written, self::MAX_WRITE_BUFFER)); + } else { + // write remaining buffer bytes to stream + $wrote = $this->stream->send(substr($buf, $written)); + } + if ($wrote === -1 || $wrote === false) { + throw new \Kafka\Exception\Socket('Could not write ' . strlen($buf) . ' bytes to stream, completed writing only ' . $written . ' bytes'); + } elseif ($wrote === 0) { + // Increment the number of times we have failed + $failedWriteAttempts++; + if ($failedWriteAttempts > $this->maxWriteAttempts) { + throw new \Kafka\Exception\Socket('After ' . $failedWriteAttempts . ' attempts could not write ' . strlen($buf) . ' bytes to stream, completed writing only ' . $written . ' bytes'); + } + } else { + // If we wrote something, reset our failed attempt counter + $failedWriteAttempts = 0; + } + $written += $wrote; + } + return $written; + } + + // }}} + // {{{ public function rewind() + + /** + * Rewind the stream + * + * @return void + */ + public function rewind() + { +// if (is_resource($this->stream)) { +// rewind($this->stream); +// } + } + + // }}} + // }}} +} diff --git a/composer.json b/composer.json index e809095e..545151d4 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,8 @@ "ext-memcached": "*", "ext-inotify": "*", "ext-curl": "*", - "ext-openssl": "*" + "ext-openssl": "*", + "amphp/amp": "^2.5" }, "autoload": { "psr-4": { @@ -34,7 +35,8 @@ "Console\\": "Console/", "Database\\": "Database/", "Queue\\": "Queue/", - "Gii\\": "Gii/" + "Gii\\": "Gii/", + "Kafka\\": "Kafka/" }, "files": [ "error.php",