From ed7cbc0e7135fcbad6ee3763e6236f5cf8d5a25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mr=C2=B7x?= Date: Wed, 28 Oct 2020 15:47:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Kafka/Broker.php | 247 --------- Kafka/Config.php | 193 ------- Kafka/Consumer.php | 71 --- Kafka/Consumer/Assignment.php | 301 ----------- Kafka/Consumer/Process.php | 812 ---------------------------- Kafka/Consumer/State.php | 459 ---------------- Kafka/ConsumerConfig.php | 149 ----- Kafka/Exception.php | 31 -- Kafka/Exception/Config.php | 33 -- Kafka/Exception/NotSupported.php | 33 -- Kafka/Exception/Protocol.php | 33 -- Kafka/Exception/Socket.php | 9 - Kafka/KafkaProviders.php | 3 + Kafka/Producer.php | 147 ++--- Kafka/Producer/Process.php | 384 ------------- Kafka/Producer/State.php | 275 ---------- Kafka/Producer/SyncProcess.php | 198 ------- Kafka/ProducerConfig.php | 102 ---- Kafka/Protocol.php | 294 ---------- Kafka/Protocol/CommitOffset.php | 226 -------- Kafka/Protocol/DescribeGroups.php | 201 ------- Kafka/Protocol/Fetch.php | 377 ------------- Kafka/Protocol/FetchOffset.php | 185 ------- Kafka/Protocol/GroupCoordinator.php | 85 --- Kafka/Protocol/Heartbeat.php | 85 --- Kafka/Protocol/JoinGroup.php | 204 ------- Kafka/Protocol/LeaveGroup.php | 79 --- Kafka/Protocol/ListGroup.php | 99 ---- Kafka/Protocol/Metadata.php | 179 ------ Kafka/Protocol/Offset.php | 203 ------- Kafka/Protocol/Produce.php | 276 ---------- Kafka/Protocol/Protocol.php | 679 ----------------------- Kafka/Protocol/SyncGroup.php | 212 -------- Kafka/SingletonTrait.php | 76 --- Kafka/Socket.php | 396 -------------- Kafka/SocketSync.php | 337 ------------ 36 files changed, 43 insertions(+), 7630 deletions(-) delete mode 100644 Kafka/Broker.php delete mode 100644 Kafka/Config.php delete mode 100644 Kafka/Consumer.php delete mode 100644 Kafka/Consumer/Assignment.php delete mode 100644 Kafka/Consumer/Process.php delete mode 100644 Kafka/Consumer/State.php delete mode 100644 Kafka/ConsumerConfig.php delete mode 100644 Kafka/Exception.php delete mode 100644 Kafka/Exception/Config.php delete mode 100644 Kafka/Exception/NotSupported.php delete mode 100644 Kafka/Exception/Protocol.php delete mode 100644 Kafka/Exception/Socket.php delete mode 100644 Kafka/Producer/Process.php delete mode 100644 Kafka/Producer/State.php delete mode 100644 Kafka/Producer/SyncProcess.php delete mode 100644 Kafka/ProducerConfig.php delete mode 100644 Kafka/Protocol.php delete mode 100644 Kafka/Protocol/CommitOffset.php delete mode 100644 Kafka/Protocol/DescribeGroups.php delete mode 100644 Kafka/Protocol/Fetch.php delete mode 100644 Kafka/Protocol/FetchOffset.php delete mode 100644 Kafka/Protocol/GroupCoordinator.php delete mode 100644 Kafka/Protocol/Heartbeat.php delete mode 100644 Kafka/Protocol/JoinGroup.php delete mode 100644 Kafka/Protocol/LeaveGroup.php delete mode 100644 Kafka/Protocol/ListGroup.php delete mode 100644 Kafka/Protocol/Metadata.php delete mode 100644 Kafka/Protocol/Offset.php delete mode 100644 Kafka/Protocol/Produce.php delete mode 100644 Kafka/Protocol/Protocol.php delete mode 100644 Kafka/Protocol/SyncGroup.php delete mode 100644 Kafka/SingletonTrait.php delete mode 100644 Kafka/Socket.php delete mode 100644 Kafka/SocketSync.php diff --git a/Kafka/Broker.php b/Kafka/Broker.php deleted file mode 100644 index 0a6955ac..00000000 --- a/Kafka/Broker.php +++ /dev/null @@ -1,247 +0,0 @@ -process = $process; - } - - // }}} - // {{{ public function setGroupBrokerId() - - public function setGroupBrokerId($brokerId) - { - $this->groupBrokerId = $brokerId; - } - - // }}} - // {{{ public function getGroupBrokerId() - - public function getGroupBrokerId() - { - return $this->groupBrokerId; - } - - // }}} - // {{{ public function setData() - - public function setData($topics, $brokersResult) - { - $brokers = array(); - foreach ($brokersResult as $value) { - $key = $value['nodeId']; - $hostname = $value['host'] . ':' . $value['port']; - $brokers[$key] = $hostname; - } - - $change = false; - if (serialize($this->brokers) != serialize($brokers)) { - $this->brokers = $brokers; - $change = true; - } - - $newTopics = array(); - foreach ($topics as $topic) { - if ($topic['errorCode'] != \Kafka\Protocol::NO_ERROR) { - $this->error('Parse metadata for topic is error, error:' . \Kafka\Protocol::getError($topic['errorCode'])); - continue; - } - $item = array(); - foreach ($topic['partitions'] as $part) { - $item[$part['partitionId']] = $part['leader']; - } - $newTopics[$topic['topicName']] = $item; - } - - if (serialize($this->topics) != serialize($newTopics)) { - $this->topics = $newTopics; - $change = true; - } - - return $change; - } - - // }}} - // {{{ public function getTopics() - - public function getTopics() - { - return $this->topics; - } - - // }}} - // {{{ public function getBrokers() - - public function getBrokers() - { - return $this->brokers; - } - - // }}} - // {{{ public function getMetaConnect() - - public function getMetaConnect($key, $modeSync = false) - { - return $this->getConnect($key, 'metaSockets', $modeSync); - } - - // }}} - // {{{ public function getRandConnect() - - public function getRandConnect($modeSync = false) - { - $nodeIds = array_keys($this->brokers); - shuffle($nodeIds); - if (!isset($nodeIds[0])) { - return false; - } - return $this->getMetaConnect($nodeIds[0], $modeSync); - } - - // }}} - // {{{ public function getDataConnect() - - public function getDataConnect($key, $modeSync = false) - { - return $this->getConnect($key, 'dataSockets', $modeSync); - } - - // }}} - // {{{ public function getConnect() - - public function getConnect($key, $type, $modeSync = false) - { - if (isset($this->{$type}[$key])) { - return $this->{$type}[$key]; - } - - if (isset($this->brokers[$key])) { - $hostname = $this->brokers[$key]; - if (isset($this->{$type}[$hostname])) { - return $this->{$type}[$hostname]; - } - } - - $host = null; - $port = null; - if (isset($this->brokers[$key])) { - $hostname = $this->brokers[$key]; - list($host, $port) = explode(':', $hostname); - } - - if (strpos($key, ':') !== false) { - list($host, $port) = explode(':', $key); - } - - if ($host && $port) { - try { - $socket = $this->getSocket($host, $port, $modeSync); - if (!$modeSync) { - $socket->SetonReadable($this->process); - } - $socket->connect(); - $this->{$type}[$key] = $socket; - return $socket; - } catch (\Exception $e) { - $this->error($e->getMessage()); - $this->error($e->getFile() . ':' . $e->getLine()); - return false; - } - } else { - return false; - } - } - - // }}} - // {{{ public function clear() - - public function clear() - { - foreach ($this->metaSockets as $key => $socket) { - $socket->close(); - } - foreach ($this->dataSockets as $key => $socket) { - $socket->close(); - } - $this->brokers = []; - } - - // }}} - // {{{ public function getSocket() - - public function getSocket($host, $port, $modeSync) - { - if ($this->socket != null) { - return $this->socket; - } - - if ($modeSync) { - $socket = new \Kafka\SocketSync($host, $port); - } else { - $socket = new \Kafka\Socket($host, $port); - } - return $socket; - } - - // }}} - // {{{ public function setSocket() - - // use unit test - public function setSocket($socket) - { - $this->socket = $socket; - } - - // }}} - // }}} -} diff --git a/Kafka/Config.php b/Kafka/Config.php deleted file mode 100644 index 763aa124..00000000 --- a/Kafka/Config.php +++ /dev/null @@ -1,193 +0,0 @@ - 'kafka-php', - 'brokerVersion' => '0.10.1.0', - 'metadataBrokerList' => '', - 'messageMaxBytes' => '1000000', - 'metadataRequestTimeoutMs' => '60000', - 'metadataRefreshIntervalMs' => '300000', - 'metadataMaxAgeMs' => -1, - ); - - // }}} - // {{{ functions - // {{{ public function __call() - - public function __call($name, $args) - { - if (strpos($name, 'get') === 0 || strpos($name, 'iet') === 0) { - $option = strtolower(substr($name, 3, 1)) . substr($name, 4); - if (isset(self::$options[$option])) { - return self::$options[$option]; - } - - if (isset(self::$defaults[$option])) { - return self::$defaults[$option]; - } - if (isset(static::$defaults[$option])) { - return static::$defaults[$option]; - } - return false; - } - - if (strpos($name, 'set') === 0) { - if (count($args) != 1) { - return false; - } - $option = strtolower(substr($name, 3, 1)) . substr($name, 4); - static::$options[$option] = $args[0]; - // check todo - return true; - } - } - - // }}} - // {{{ public function setClientId() - - public function setClientId($val) - { - $client = trim($val); - if ($client == '') { - throw new \Kafka\Exception\Config('Set clientId value is invalid, must is not empty string.'); - } - static::$options['clientId'] = $client; - } - - // }}} - // {{{ public function setBrokerVersion() - - public function setBrokerVersion($version) - { - $version = trim($version); - if ($version == '' || version_compare($version, '0.8.0') < 0) { - throw new \Kafka\Exception\Config('Set broker version value is invalid, must is not empty string and gt 0.8.0.'); - } - static::$options['brokerVersion'] = $version; - } - - // }}} - // {{{ public function setMetadataBrokerList() - - public function setMetadataBrokerList($list) - { - if (trim($list) == '') { - throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); - } - $tmp = explode(',', trim($list)); - $lists = array(); - foreach ($tmp as $key => $val) { - if (trim($val) != '') { - $lists[] = $val; - } - } - if (empty($lists)) { - throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); - } - foreach ($lists as $val) { - $hostinfo = explode(':', $val); - foreach ($hostinfo as $key => $val) { - if (trim($val) == '') { - unset($hostinfo[$key]); - } - } - if (count($hostinfo) != 2) { - throw new \Kafka\Exception\Config('Set broker list value is invalid, must is not empty string'); - } - } - - static::$options['metadataBrokerList'] = $list; - } - - // }}} - // {{{ public function clear() - - public function clear() - { - static::$options = []; - } - - // }}} - // {{{ public function setMessageMaxBytes() - - public function setMessageMaxBytes($messageMaxBytes) - { - if (!is_numeric($messageMaxBytes) || $messageMaxBytes < 1000 || $messageMaxBytes > 1000000000) { - throw new \Kafka\Exception\Config('Set message max bytes value is invalid, must set it 1000 .. 1000000000'); - } - static::$options['messageMaxBytes'] = $messageMaxBytes; - } - - // }}} - // {{{ public function setMetadataRequestTimeoutMs() - - public function setMetadataRequestTimeoutMs($metadataRequestTimeoutMs) - { - if (!is_numeric($metadataRequestTimeoutMs) || $metadataRequestTimeoutMs < 10 - || $metadataRequestTimeoutMs > 900000) { - throw new \Kafka\Exception\Config('Set metadata request timeout value is invalid, must set it 10 .. 900000'); - } - static::$options['metadataRequestTimeoutMs'] = $metadataRequestTimeoutMs; - } - - // }}} - // {{{ public function setMetadataRefreshIntervalMs() - - public function setMetadataRefreshIntervalMs($metadataRefreshIntervalMs) - { - if (!is_numeric($metadataRefreshIntervalMs) || $metadataRefreshIntervalMs < 10 - || $metadataRefreshIntervalMs > 3600000) { - throw new \Kafka\Exception\Config('Set metadata refresh interval value is invalid, must set it 10 .. 3600000'); - } - static::$options['metadataRefreshIntervalMs'] = $metadataRefreshIntervalMs; - } - - // }}} - // {{{ public function setMetadataMaxAgeMs() - - public function setMetadataMaxAgeMs($metadataMaxAgeMs) - { - if (!is_numeric($metadataMaxAgeMs) || $metadataMaxAgeMs < 1 - || $metadataMaxAgeMs > 86400000) { - throw new \Kafka\Exception\Config('Set metadata max age value is invalid, must set it 1 .. 86400000'); - } - static::$options['metadataMaxAgeMs'] = $metadataMaxAgeMs; - } - - // }}} - // }}} -} diff --git a/Kafka/Consumer.php b/Kafka/Consumer.php deleted file mode 100644 index 249ed295..00000000 --- a/Kafka/Consumer.php +++ /dev/null @@ -1,71 +0,0 @@ -isRunning) { - $this->error('Has start consumer'); - return; - } - $process = new \Kafka\Consumer\Process($consumer); - if ($this->logger) { - $process->setLogger($this->logger); - } - $process->start(); - $this->isRunning = true; - if ($isBlock) { - run(); - } - } -} diff --git a/Kafka/Consumer/Assignment.php b/Kafka/Consumer/Assignment.php deleted file mode 100644 index 1165437c..00000000 --- a/Kafka/Consumer/Assignment.php +++ /dev/null @@ -1,301 +0,0 @@ -memberId = $memberId; - } - - // }}} - // {{{ public function getMemberId() - - public function getMemberId() - { - return $this->memberId; - } - - // }}} - // {{{ public function setGenerationId() - - public function setGenerationId($generationId) - { - $this->generationId = $generationId; - } - - // }}} - // {{{ public function getGenerationId() - - public function getGenerationId() - { - return $this->generationId; - } - - // }}} - // {{{ public function getAssignments() - - public function getAssignments() - { - return $this->assignments; - } - - // }}} - // {{{ public function assign() - - public function assign($result) - { - $broker = \Kafka\Broker::getInstance(); - $topics = $broker->getTopics(); - - $memberCount = count($result); - - $count = 0; - $members = array(); - foreach ($topics as $topicName => $partition) { - foreach ($partition as $partId => $leaderId) { - $memberNum = $count % $memberCount; - if (!isset($members[$memberNum])) { - $members[$memberNum] = array(); - } - if (!isset($members[$memberNum][$topicName])) { - $members[$memberNum][$topicName] = array(); - } - $members[$memberNum][$topicName]['topic_name'] = $topicName; - if (!isset($members[$memberNum][$topicName]['partitions'])) { - $members[$memberNum][$topicName]['partitions'] = array(); - } - $members[$memberNum][$topicName]['partitions'][] = $partId; - $count++; - } - } - - $data = array(); - foreach ($result as $key => $member) { - $item = array( - 'version' => 0, - 'member_id' => $member['memberId'], - 'assignments' => isset($members[$key]) ? $members[$key] : array() - ); - $data[] = $item; - } - $this->assignments = $data; - } - - // }}} - // {{{ public function setTopics() - - public function setTopics($topics) - { - $this->topics = $topics; - } - - // }}} - // {{{ public function getTopics() - - public function getTopics() - { - return $this->topics; - } - - // }}} - // {{{ public function setOffsets() - - public function setOffsets($offsets) - { - $this->offsets = $offsets; - } - - // }}} - // {{{ public function getOffsets() - - public function getOffsets() - { - return $this->offsets; - } - - // }}} - // {{{ public function setLastOffsets() - - public function setLastOffsets($offsets) - { - $this->lastOffsets = $offsets; - } - - // }}} - // {{{ public function getOffsets() - - public function getLastOffsets() - { - return $this->lastOffsets; - } - - // }}} - // {{{ public function setFetchOffsets() - - public function setFetchOffsets($offsets) - { - $this->fetchOffsets = $offsets; - } - - // }}} - // {{{ public function getFetchOffsets() - - public function getFetchOffsets() - { - return $this->fetchOffsets; - } - - // }}} - // {{{ public function setConsumerOffsets() - - public function setConsumerOffsets($offsets) - { - $this->consumerOffsets = $offsets; - } - - // }}} - // {{{ public function getConsumerOffsets() - - public function getConsumerOffsets() - { - return $this->consumerOffsets; - } - - // }}} - // {{{ public function setConsumerOffset() - - public function setConsumerOffset($topic, $part, $offset) - { - $this->consumerOffsets[$topic][$part] = $offset; - } - - // }}} - // {{{ public function getConsumerOffset() - - public function getConsumerOffset($topic, $part) - { - if (!isset($this->consumerOffsets[$topic][$part])) { - return false; - } - return $this->consumerOffsets[$topic][$part]; - } - - // }}} - // {{{ public function setCommitOffsets() - - public function setCommitOffsets($offsets) - { - $this->commitOffsets = $offsets; - } - - // }}} - // {{{ public function getCommitOffsets() - - public function getCommitOffsets() - { - return $this->commitOffsets; - } - - // }}} - // {{{ public function setCommitOffset() - - public function setCommitOffset($topic, $part, $offset) - { - $this->commitOffsets[$topic][$part] = $offset; - } - - // }}} - // {{{ public function setPrecommitOffsets() - - public function setPrecommitOffsets($offsets) - { - $this->precommitOffsets = $offsets; - } - - // }}} - // {{{ public function getPrecommitOffsets() - - public function getPrecommitOffsets() - { - return $this->precommitOffsets; - } - - // }}} - // {{{ public function setPrecommitOffset() - - public function setPrecommitOffset($topic, $part, $offset) - { - $this->precommitOffsets[$topic][$part] = $offset; - } - - // }}} - // {{{ public function clearOffset() - - public function clearOffset() - { - $this->offsets = array(); - $this->lastOffsets = array(); - $this->fetchOffsets = array(); - $this->consumerOffsets = array(); - $this->commitOffsets = array(); - $this->precommitOffsets = array(); - } - - // }}} - // }}} -} diff --git a/Kafka/Consumer/Process.php b/Kafka/Consumer/Process.php deleted file mode 100644 index bb5ff167..00000000 --- a/Kafka/Consumer/Process.php +++ /dev/null @@ -1,812 +0,0 @@ -consumer = $consumer; - } - - // }}} - // {{{ public function init() - - /** - * start consumer - * - * @access public - * @return void - */ - public function init() - { - // init protocol - $config = \Kafka\ConsumerConfig::getInstance(); - \Kafka\Protocol::init($config->getBrokerVersion(), $this->logger); - - // init process request - $broker = \Kafka\Broker::getInstance(); - $broker->setProcess(function ($data, $fd) { - $this->processRequest($data, $fd); - }); - - // init state - $this->state = \Kafka\Consumer\State::getInstance(); - if ($this->logger) { - $this->state->setLogger($this->logger); - } - $this->state->setCallback(array( - \Kafka\Consumer\State::REQUEST_METADATA => function () { - return $this->syncMeta(); - }, - \Kafka\Consumer\State::REQUEST_GETGROUP => function () { - return $this->getGroupBrokerId(); - }, - \Kafka\Consumer\State::REQUEST_JOINGROUP => function () { - return $this->joinGroup(); - }, - \Kafka\Consumer\State::REQUEST_SYNCGROUP => function () { - return $this->syncGroup(); - }, - \Kafka\Consumer\State::REQUEST_HEARTGROUP => function () { - return $this->heartbeat(); - }, - \Kafka\Consumer\State::REQUEST_OFFSET => function () { - return $this->offset(); - }, - \Kafka\Consumer\State::REQUEST_FETCH_OFFSET => function () { - return $this->fetchOffset(); - }, - \Kafka\Consumer\State::REQUEST_FETCH => function () { - return $this->fetch(); - }, - \Kafka\Consumer\State::REQUEST_COMMIT_OFFSET => function () { - return $this->commit(); - }, - )); - $this->state->init(); - } - - // }}} - // {{{ public function start() - - /** - * start consumer - * - * @access public - * @return void - */ - public function start() - { - $this->init(); - $this->state->start(); - } - - // }}} - // {{{ public function stop() - - /** - * stop consumer - * - * @access public - * @return void - */ - public function stop() - { - $this->isRunning = false; - } - - // }}} - // {{{ protected function processRequest() - - /** - * process Request - * - * @access public - * @return void - */ - protected function processRequest($data, $fd) - { - $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); - switch ($correlationId) { - case \Kafka\Protocol::METADATA_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); - if (!isset($result['brokers']) || !isset($result['topics'])) { - $this->error('Get metadata is fail, brokers or topics is null.'); - $this->state->failRun(\Kafka\Consumer\State::REQUEST_METADATA); - } else { - $broker = \Kafka\Broker::getInstance(); - $isChange = $broker->setData($result['topics'], $result['brokers']); - $this->state->succRun(\Kafka\Consumer\State::REQUEST_METADATA, $isChange); - } - break; - case \Kafka\Protocol::GROUP_COORDINATOR_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::GROUP_COORDINATOR_REQUEST, substr($data, 4)); - if (isset($result['errorCode']) && $result['errorCode'] == \Kafka\Protocol::NO_ERROR - && isset($result['coordinatorId'])) { - \Kafka\Broker::getInstance()->setGroupBrokerId($result['coordinatorId']); - $this->state->succRun(\Kafka\Consumer\State::REQUEST_GETGROUP); - } else { - $this->state->failRun(\Kafka\Consumer\State::REQUEST_GETGROUP); - } - break; - case \Kafka\Protocol::JOIN_GROUP_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::JOIN_GROUP_REQUEST, substr($data, 4)); - if (isset($result['errorCode']) && $result['errorCode'] == 0) { - $this->succJoinGroup($result); - } else { - $this->failJoinGroup($result['errorCode']); - } - break; - case \Kafka\Protocol::SYNC_GROUP_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::SYNC_GROUP_REQUEST, substr($data, 4)); - if (isset($result['errorCode']) && $result['errorCode'] == 0) { - $this->succSyncGroup($result); - } else { - $this->failSyncGroup($result['errorCode']); - } - break; - case \Kafka\Protocol::HEART_BEAT_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::HEART_BEAT_REQUEST, substr($data, 4)); - if (isset($result['errorCode']) && $result['errorCode'] == 0) { - $this->state->succRun(\Kafka\Consumer\State::REQUEST_HEARTGROUP); - } else { - $this->failHeartbeat($result['errorCode']); - } - break; - case \Kafka\Protocol::OFFSET_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_REQUEST, substr($data, 4)); - $this->succOffset($result, $fd); - break; - case \Kafka\Protocol\Protocol::OFFSET_FETCH_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_FETCH_REQUEST, substr($data, 4)); - $this->succFetchOffset($result); - break; - case \Kafka\Protocol\Protocol::FETCH_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::FETCH_REQUEST, substr($data, 4)); - $this->succFetch($result, $fd); - break; - case \Kafka\Protocol\Protocol::OFFSET_COMMIT_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::OFFSET_COMMIT_REQUEST, substr($data, 4)); - $this->succCommit($result); - break; - default: - $this->error('Error request, correlationId:' . $correlationId); - } - } - - // }}} - // {{{ protected function syncMeta() - - protected function syncMeta() - { - $this->debug('Start sync metadata request'); - $brokerList = explode(',', \Kafka\ConsumerConfig::getInstance()->getMetadataBrokerList()); - $brokerHost = array(); - foreach ($brokerList as $key => $val) { - if (trim($val)) { - $brokerHost[] = $val; - } - } - if (count($brokerHost) == 0) { - throw new \Kafka\Exception('Not set config `metadataBrokerList`'); - } - shuffle($brokerHost); - $broker = \Kafka\Broker::getInstance(); - foreach ($brokerHost as $host) { - $socket = $broker->getMetaConnect($host); - if ($socket) { - $params = \Kafka\ConsumerConfig::getInstance()->getTopics(); - $this->debug('Start sync metadata request params:' . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); - $socket->write($requestData); - return; - } - } - throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); - } - - // }}} - // {{{ protected function getGroupBrokerId() - - protected function getGroupBrokerId() - { - $broker = \Kafka\Broker::getInstance(); - $connect = $broker->getRandConnect(); - if (!$connect) { - return; - } - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - ); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::GROUP_COORDINATOR_REQUEST, $params); - $connect->write($requestData); - } - - // }}} - // {{{ protected function joinGroup() - - protected function joinGroup() - { - $broker = \Kafka\Broker::getInstance(); - $groupBrokerId = $broker->getGroupBrokerId(); - $connect = $broker->getMetaConnect($groupBrokerId); - if (!$connect) { - return false; - } - $topics = \Kafka\ConsumerConfig::getInstance()->getTopics(); - $assign = \Kafka\Consumer\Assignment::getInstance(); - $memberId = $assign->getMemberId(); - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - 'session_timeout' => \Kafka\ConsumerConfig::getInstance()->getSessionTimeout(), - 'rebalance_timeout' => \Kafka\ConsumerConfig::getInstance()->getRebalanceTimeout(), - 'member_id' => ($memberId == null) ? '' : $memberId, - 'data' => array( - array( - 'protocol_name' => 'range', - 'version' => 0, - 'subscription' => $topics, - 'user_data' => '', - ), - ), - ); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::JOIN_GROUP_REQUEST, $params); - $connect->write($requestData); - $this->debug("Join group start, params:" . json_encode($params)); - } - - // }}} - // {{{ public function failJoinGroup() - - public function failJoinGroup($errorCode) - { - $assign = \Kafka\Consumer\Assignment::getInstance(); - $memberId = $assign->getMemberId(); - $error = sprintf('Join group fail, need rejoin, errorCode %d, memberId: %s', $errorCode, $memberId); - $this->error($error); - $this->stateConvert($errorCode); - } - - // }}} - // {{{ public function succJoinGroup() - - public function succJoinGroup($result) - { - $this->state->succRun(\Kafka\Consumer\State::REQUEST_JOINGROUP); - $assign = \Kafka\Consumer\Assignment::getInstance(); - $assign->setMemberId($result['memberId']); - $assign->setGenerationId($result['generationId']); - if ($result['leaderId'] == $result['memberId']) { // leader assign partition - $assigns = $assign->assign($result['members']); - } - $msg = sprintf('Join group sucess, params: %s', json_encode($result)); - $this->debug($msg); - } - - // }}} - // {{{ public function syncGroup() - - public function syncGroup() - { - $broker = \Kafka\Broker::getInstance(); - $groupBrokerId = $broker->getGroupBrokerId(); - $connect = $broker->getMetaConnect($groupBrokerId); - if (!$connect) { - return; - } - $topics = \Kafka\ConsumerConfig::getInstance()->getTopics(); - $assign = \Kafka\Consumer\Assignment::getInstance(); - $memberId = $assign->getMemberId(); - $generationId = $assign->getGenerationId(); - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - 'generation_id' => $generationId, - 'member_id' => $memberId, - 'data' => $assign->getAssignments(), - ); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::SYNC_GROUP_REQUEST, $params); - $this->debug("Sync group start, params:" . json_encode($params)); - $connect->write($requestData); - } - - // }}} - // {{{ public function failSyncGroup() - - public function failSyncGroup($errorCode) - { - $error = sprintf('Sync group fail, need rejoin, errorCode %d', $errorCode); - $this->error($error); - $this->stateConvert($errorCode); - } - - // }}} - // {{{ public function succSyncGroup() - - public function succSyncGroup($result) - { - $msg = sprintf('Sync group sucess, params: %s', json_encode($result)); - $this->debug($msg); - $this->state->succRun(\Kafka\Consumer\State::REQUEST_SYNCGROUP); - - $topics = \Kafka\Broker::getInstance()->getTopics(); - $brokerToTopics = array(); - foreach ($result['partitionAssignments'] as $topic) { - foreach ($topic['partitions'] as $partId) { - $brokerId = $topics[$topic['topicName']][$partId]; - if (!isset($brokerToTopics[$brokerId])) { - $brokerToTopics[$brokerId] = array(); - } - - $topicInfo = array(); - if (isset($brokerToTopics[$brokerId][$topic['topicName']])) { - $topicInfo = $brokerToTopics[$brokerId][$topic['topicName']]; - } - $topicInfo['topic_name'] = $topic['topicName']; - if (!isset($topicInfo['partitions'])) { - $topicInfo['partitions'] = array(); - } - $topicInfo['partitions'][] = $partId; - $brokerToTopics[$brokerId][$topic['topicName']] = $topicInfo; - } - } - $assign = \Kafka\Consumer\Assignment::getInstance(); - $assign->setTopics($brokerToTopics); - } - - // }}} - // {{{ protected function heartbeat() - - protected function heartbeat() - { - $broker = \Kafka\Broker::getInstance(); - $groupBrokerId = $broker->getGroupBrokerId(); - $connect = $broker->getMetaConnect($groupBrokerId); - if (!$connect) { - return; - } - $assign = \Kafka\Consumer\Assignment::getInstance(); - $memberId = $assign->getMemberId(); - if (!$memberId) { - return; - } - $generationId = $assign->getGenerationId(); - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - 'generation_id' => $generationId, - 'member_id' => $memberId, - ); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::HEART_BEAT_REQUEST, $params); - //$this->debug("Heartbeat group start, params:" . json_encode($params)); - $connect->write($requestData); - } - -// }}} - // {{{ public function failHeartbeat() - - public function failHeartbeat($errorCode) - { - $this->error('Heartbeat error, errorCode:' . $errorCode); - $this->stateConvert($errorCode); - } - - // }}} - // {{{ protected function offset() - - protected function offset() - { - $context = array(); - $broker = \Kafka\Broker::getInstance(); - $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); - foreach ($topics as $brokerId => $topicList) { - $connect = $broker->getMetaConnect($brokerId); - if (!$connect) { - return; - } - $data = array(); - foreach ($topicList as $topic) { - $item = array( - 'topic_name' => $topic['topic_name'], - 'partitions' => array(), - ); - foreach ($topic['partitions'] as $partId) { - $item['partitions'][] = array( - 'partition_id' => $partId, - 'offset' => 1, - 'time' => -1, - ); - $data[] = $item; - } - } - $params = array( - 'replica_id' => -1, - 'data' => $data, - ); - $stream = $connect->getSocket(); - //$this->debug("Get current offset start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_REQUEST, $params); - $connect->write($requestData); - $context[] = (int)$stream; - } - - return $context; - } - - // }}} - // {{{ public function succOffset() - - public function succOffset($result, $fd) - { - $msg = sprintf('Get current offset sucess, result: %s', json_encode($result)); - //$this->debug($msg); - - $offsets = \Kafka\Consumer\Assignment::getInstance()->getOffsets(); - $lastOffsets = \Kafka\Consumer\Assignment::getInstance()->getLastOffsets(); - foreach ($result as $topic) { - foreach ($topic['partitions'] as $part) { - if ($part['errorCode'] != 0) { - $this->stateConvert($part['errorCode']); - break 2; - } - - $offsets[$topic['topicName']][$part['partition']] = end($part['offsets']); - $lastOffsets[$topic['topicName']][$part['partition']] = $part['offsets'][0]; - } - } - \Kafka\Consumer\Assignment::getInstance()->setOffsets($offsets); - \Kafka\Consumer\Assignment::getInstance()->setLastOffsets($lastOffsets); - $this->state->succRun(\Kafka\Consumer\State::REQUEST_OFFSET, $fd); - } - - // }}} - // {{{ protected function fetchOffset() - - protected function fetchOffset() - { - $broker = \Kafka\Broker::getInstance(); - $groupBrokerId = $broker->getGroupBrokerId(); - $connect = $broker->getMetaConnect($groupBrokerId); - if (!$connect) { - return; - } - - $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); - $data = array(); - foreach ($topics as $brokerId => $topicList) { - foreach ($topicList as $topic) { - $partitions = array(); - if (isset($data[$topic['topic_name']]['partitions'])) { - $partitions = $data[$topic['topic_name']]['partitions']; - } - foreach ($topic['partitions'] as $partId) { - $partitions[] = $partId; - } - $data[$topic['topic_name']]['partitions'] = $partitions; - $data[$topic['topic_name']]['topic_name'] = $topic['topic_name']; - } - } - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - 'data' => $data, - ); - //$this->debug("Get current fetch offset start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_FETCH_REQUEST, $params); - $connect->write($requestData); - } - - // }}} - // {{{ public function succFetchOffset() - - public function succFetchOffset($result) - { - $msg = sprintf('Get current fetch offset sucess, result: %s', json_encode($result)); - $this->debug($msg); - - $assign = \Kafka\Consumer\Assignment::getInstance(); - $offsets = $assign->getFetchOffsets(); - foreach ($result as $topic) { - foreach ($topic['partitions'] as $part) { - if ($part['errorCode'] != 0) { - $this->stateConvert($part['errorCode']); - break 2; - } - - $offsets[$topic['topicName']][$part['partition']] = $part['offset']; - } - } - $assign->setFetchOffsets($offsets); - - $consumerOffsets = $assign->getConsumerOffsets(); - $lastOffsets = $assign->getLastOffsets(); - if (empty($consumerOffsets)) { - $consumerOffsets = $assign->getFetchOffsets(); - foreach ($consumerOffsets as $topic => $value) { - foreach ($value as $partId => $offset) { - if (isset($lastOffsets[$topic][$partId]) && $lastOffsets[$topic][$partId] > $offset) { - $consumerOffsets[$topic][$partId] = $offset + 1; - } - } - } - $assign->setConsumerOffsets($consumerOffsets); - $assign->setCommitOffsets($assign->getFetchOffsets()); - } - $this->state->succRun(\Kafka\Consumer\State::REQUEST_FETCH_OFFSET); - } - - // }}} - // {{{ protected function fetch() - - protected function fetch() - { - $this->messages = array(); - $context = array(); - $broker = \Kafka\Broker::getInstance(); - $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); - $consumerOffsets = \Kafka\Consumer\Assignment::getInstance()->getConsumerOffsets(); - foreach ($topics as $brokerId => $topicList) { - $connect = $broker->getDataConnect($brokerId); - if (!$connect) { - return; - } - - $data = array(); - foreach ($topicList as $topic) { - $item = array( - 'topic_name' => $topic['topic_name'], - 'partitions' => array(), - ); - foreach ($topic['partitions'] as $partId) { - $item['partitions'][] = array( - 'partition_id' => $partId, - 'offset' => isset($consumerOffsets[$topic['topic_name']][$partId]) ? $consumerOffsets[$topic['topic_name']][$partId] : 0, - 'max_bytes' => \Kafka\ConsumerConfig::getInstance()->getMaxBytes(), - ); - } - $data[] = $item; - } - $params = array( - 'max_wait_time' => \Kafka\ConsumerConfig::getInstance()->getMaxWaitTime(), - 'replica_id' => -1, - 'min_bytes' => '1000', - 'data' => $data, - ); - $this->debug("Fetch message start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::FETCH_REQUEST, $params); - $connect->write($requestData); - $context[] = (int)$connect->getSocket(); - } - return $context; - } - - // }}} - // {{{ public function succFetch() - - public function succFetch($result, $fd) - { - $assign = \Kafka\Consumer\Assignment::getInstance(); - $this->debug('Fetch success, result:' . json_encode($result)); - foreach ($result['topics'] as $topic) { - foreach ($topic['partitions'] as $part) { - $context = array( - $topic['topicName'], - $part['partition'], - ); - if ($part['errorCode'] != 0) { - $this->stateConvert($part['errorCode'], $context); - continue; - } - - $offset = $assign->getConsumerOffset($topic['topicName'], $part['partition']); - if ($offset === false) { - return; // current is rejoin.... - } - foreach ($part['messages'] as $message) { - $this->messages[$topic['topicName']][$part['partition']][] = $message; - //if ($this->consumer != null) { - // call_user_func($this->consumer, $topic['topicName'], $part['partition'], $message); - //} - $offset = $message['offset']; - } - - $consumerOffset = ($part['highwaterMarkOffset'] > $offset) ? ($offset + 1) : $offset; - $assign->setConsumerOffset($topic['topicName'], $part['partition'], $consumerOffset); - $assign->setCommitOffset($topic['topicName'], $part['partition'], $offset); - } - } - $this->state->succRun(\Kafka\Consumer\State::REQUEST_FETCH, $fd); - } - - // }}} - // {{{ protected function commit() - - protected function consume_msg() - { - foreach ($this->messages as $topic => $value) { - foreach ($value as $part => $messages) { - foreach ($messages as $message) { - if ($this->consumer != null) { - call_user_func($this->consumer, $topic, $part, $message); - } - } - } - } - - $this->messages = array(); - } - - - protected function commit() - { - $config= ConsumerConfig::getInstance(); - if($config->getConsumeMode() == ConsumerConfig::CONSUME_BEFORE_COMMIT_OFFSET) - { - $this->consume_msg(); - } - - - $broker = \Kafka\Broker::getInstance(); - $groupBrokerId = $broker->getGroupBrokerId(); - $connect = $broker->getMetaConnect($groupBrokerId); - if (!$connect) { - return; - } - - $commitOffsets = \Kafka\Consumer\Assignment::getInstance()->getCommitOffsets(); - $topics = \Kafka\Consumer\Assignment::getInstance()->getTopics(); - \Kafka\Consumer\Assignment::getInstance()->setPrecommitOffsets($commitOffsets); - $data = array(); - foreach ($topics as $brokerId => $topicList) { - foreach ($topicList as $topic) { - $partitions = array(); - if (isset($data[$topic['topic_name']]['partitions'])) { - $partitions = $data[$topic['topic_name']]['partitions']; - } - foreach ($topic['partitions'] as $partId) { - if ($commitOffsets[$topic['topic_name']][$partId] == -1) { - continue; - } - $partitions[$partId]['partition'] = $partId; - $partitions[$partId]['offset'] = $commitOffsets[$topic['topic_name']][$partId]; - } - $data[$topic['topic_name']]['partitions'] = $partitions; - $data[$topic['topic_name']]['topic_name'] = $topic['topic_name']; - } - } - $params = array( - 'group_id' => \Kafka\ConsumerConfig::getInstance()->getGroupId(), - 'generation_id' => \Kafka\Consumer\Assignment::getInstance()->getGenerationId(), - 'member_id' => \Kafka\Consumer\Assignment::getInstance()->getMemberId(), - 'data' => $data, - ); - $this->debug("Commit current fetch offset start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::OFFSET_COMMIT_REQUEST, $params); - $connect->write($requestData); - } - - // }}} - // {{{ public function succCommit() - - /** - * @var State - */ - public $state; - public function succCommit($result) - { - $this->debug('Commit success, result:' . json_encode($result)); - $this->state->succRun(\Kafka\Consumer\State::REQUEST_COMMIT_OFFSET); - foreach ($result as $topic) { - foreach ($topic['partitions'] as $part) { - if ($part['errorCode'] != 0) { - $this->stateConvert($part['errorCode']); - return; // not call user consumer function - } - } - } - if(ConsumerConfig::getInstance()->getConsumeMode() == ConsumerConfig::CONSUME_AFTER_COMMIT_OFFSET) - { - $this->consume_msg(); - } - } - - // }}} - // {{{ protected function stateConvert() - - protected function stateConvert($errorCode, $context = null) - { - $retry = false; - $this->error(\Kafka\Protocol::getError($errorCode)); - $recoverCodes = array( - \Kafka\Protocol::UNKNOWN_TOPIC_OR_PARTITION, - \Kafka\Protocol::NOT_LEADER_FOR_PARTITION, - \Kafka\Protocol::BROKER_NOT_AVAILABLE, - \Kafka\Protocol::GROUP_LOAD_IN_PROGRESS, - \Kafka\Protocol::GROUP_COORDINATOR_NOT_AVAILABLE, - \Kafka\Protocol::NOT_COORDINATOR_FOR_GROUP, - \Kafka\Protocol::INVALID_TOPIC, - \Kafka\Protocol::INCONSISTENT_GROUP_PROTOCOL, - \Kafka\Protocol::INVALID_GROUP_ID, - ); - $rejoinCodes = array( - \Kafka\Protocol::ILLEGAL_GENERATION, - \Kafka\Protocol::INVALID_SESSION_TIMEOUT, - \Kafka\Protocol::REBALANCE_IN_PROGRESS, - \Kafka\Protocol::UNKNOWN_MEMBER_ID, - ); - - $assign = \Kafka\Consumer\Assignment::getInstance(); - if (in_array($errorCode, $recoverCodes)) { - $this->state->recover(); - $assign->clearOffset(); - return false; - } - - if (in_array($errorCode, $rejoinCodes)) { - if ($errorCode == \Kafka\Protocol::UNKNOWN_MEMBER_ID) { - $assign->setMemberId(''); - } - $assign->clearOffset(); - $this->state->rejoin(); - return false; - } - - if (\Kafka\Protocol::OFFSET_OUT_OF_RANGE == $errorCode) { - $resetOffset = \Kafka\ConsumerConfig::getInstance()->getOffsetReset(); - if ($resetOffset == 'latest') { - $offsets = $assign->getLastOffsets(); - } else { - $offsets = $assign->getOffsets(); - } - list($topic, $partId) = $context; - if (isset($offsets[$topic][$partId])) { - $assign->setConsumerOffset($topic, $partId, $offsets[$topic][$partId]); - } - } - return true; - } - - // }}} - // }}} -} diff --git a/Kafka/Consumer/State.php b/Kafka/Consumer/State.php deleted file mode 100644 index a61371f3..00000000 --- a/Kafka/Consumer/State.php +++ /dev/null @@ -1,459 +0,0 @@ - array(), - self::REQUEST_GETGROUP => array(), - self::REQUEST_JOINGROUP => array(), - self::REQUEST_SYNCGROUP => array(), - self::REQUEST_HEARTGROUP => array(), - self::REQUEST_OFFSET => array( - 'interval' => 2000, - ), - self::REQUEST_FETCH => array( - 'interval' => 100, - ), - self::REQUEST_FETCH_OFFSET => array( - 'interval' => 2000, - ), - self::REQUEST_COMMIT_OFFSET => array( - 'norepeat' => true, - ), - ); - - // }}} - // {{{ functions - // {{{ public function init() - - public function init() - { - $this->callStatus = array( - self::REQUEST_METADATA => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_GETGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_JOINGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_SYNCGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_HEARTGROUP => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_COMMIT_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - ); - - // instances clear - - // init requests - $config = \Kafka\ConsumerConfig::getInstance(); - foreach ($this->requests as $request => $option) { - switch ($request) { - case self::REQUEST_METADATA: - $this->requests[$request]['interval'] = $config->getMetadataRefreshIntervalMs(); - break; - default: - $this->requests[$request]['interval'] = 1000; - } - } - } - - // }}} - // {{{ public function start() - - public function start() - { - foreach ($this->requests as $request => $option) { - if (isset($option['norepeat']) && $option['norepeat']) { - continue; - } - $interval = isset($option['interval']) ? $option['interval'] : 200; - Timer::tick($msInterval = $interval, function ($watcherId) use ($request, $option) { - if ($this->checkRun($request) && $option['func'] != null) { - $context = call_user_func($option['func']); - $this->processing($request, $context); - } - $this->requests[$request]['watcher'] = $watcherId; - }); - } - - // start sync metadata - if (isset($this->requests[self::REQUEST_METADATA]['func'])) { - $context = call_user_func($this->requests[self::REQUEST_METADATA]['func']); - $this->processing($request, $context); - } - Timer::tick($msInterval = 1000, function ($watcherId) { - $this->report(); - }); - } - - // }}} - // {{{ public function succRun() - - public function succRun($key, $context = null) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - switch ($key) { - case self::REQUEST_METADATA: - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - if ($context) { // if kafka broker is change - $this->recover(); - } - break; - case self::REQUEST_GETGROUP: - case self::REQUEST_JOINGROUP: - case self::REQUEST_SYNCGROUP: - $this->callStatus[$key]['status'] = (self::STATUS_STOP | self::STATUS_FINISH); - break; - case self::REQUEST_HEARTGROUP: - case self::REQUEST_FETCH_OFFSET: - case self::REQUEST_COMMIT_OFFSET: - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - break; - case self::REQUEST_OFFSET: - if (!isset($this->callStatus[$key]['context'])) { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - break; - } - unset($this->callStatus[$key]['context'][$context]); - $contextStatus = $this->callStatus[$key]['context']; - if (empty($contextStatus)) { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - } - break; - case self::REQUEST_FETCH: - if (!isset($this->callStatus[$key]['context'])) { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - break; - } - unset($this->callStatus[$key]['context'][$context]); - $contextStatus = $this->callStatus[$key]['context']; - if (empty($contextStatus)) { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - call_user_func($this->requests[self::REQUEST_COMMIT_OFFSET]['func']); - } - break; - } - } - - // }}} - // {{{ public function failRun() - - public function failRun($key, $context = null) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - switch ($key) { - case self::REQUEST_METADATA: - $this->callStatus[$key]['status'] = self::STATUS_LOOP; - break; - case self::REQUEST_GETGROUP: - case self::REQUEST_JOINGROUP: - case self::REQUEST_SYNCGROUP: - $this->recover(); - break; - } - } - - // }}} - // {{{ public function setCallback() - - public function setCallback($callbacks) - { - foreach ($callbacks as $request => $callback) { - $this->requests[$request]['func'] = $callback; - } - } - - // }}} - // {{{ public function rejoin() - - public function rejoin() - { - $joinGroupStatus = $this->callStatus[self::REQUEST_JOINGROUP]['status']; - if (($joinGroupStatus & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return; - } - - $this->callStatus = array( - self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], - self::REQUEST_GETGROUP => $this->callStatus[self::REQUEST_GETGROUP], - self::REQUEST_JOINGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_SYNCGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_HEARTGROUP => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_COMMIT_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - ); - } - - // }}} - // {{{ public function recover() - - public function recover() - { - $this->callStatus = array( - self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], - self::REQUEST_GETGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_JOINGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_SYNCGROUP => array( - 'status' => self::STATUS_START, - ), - self::REQUEST_HEARTGROUP => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_FETCH_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - self::REQUEST_COMMIT_OFFSET => array( - 'status' => self::STATUS_LOOP, - ), - ); - } - - // }}} - // {{{ protected function checkRun() - - protected function checkRun($key) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - $status = $this->callStatus[$key]['status']; - switch ($key) { - case self::REQUEST_METADATA: - if ($status & self::STATUS_PROCESS == self::STATUS_PROCESS) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - case self::REQUEST_GETGROUP: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $metaStatus = $this->callStatus[self::REQUEST_METADATA]['status']; - if (($metaStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_START) == self::STATUS_START) { - return true; - } - return false; - case self::REQUEST_JOINGROUP: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $groupStatus = $this->callStatus[self::REQUEST_GETGROUP]['status']; - if (($groupStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_START) == self::STATUS_START) { - return true; - } - return false; - case self::REQUEST_SYNCGROUP: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $joinStatus = $this->callStatus[self::REQUEST_JOINGROUP]['status']; - if (($joinStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_START) == self::STATUS_START) { - return true; - } - return false; - case self::REQUEST_HEARTGROUP: - case self::REQUEST_OFFSET: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $syncStatus = $this->callStatus[self::REQUEST_SYNCGROUP]['status']; - if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - case self::REQUEST_FETCH_OFFSET: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $syncStatus = $this->callStatus[self::REQUEST_SYNCGROUP]['status']; - if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - $offsetStatus = $this->callStatus[self::REQUEST_OFFSET]['status']; - if (($offsetStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - case self::REQUEST_FETCH: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $fetchOffsetStatus = $this->callStatus[self::REQUEST_FETCH_OFFSET]['status']; - if (($fetchOffsetStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - $commitOffsetStatus = $this->callStatus[self::REQUEST_COMMIT_OFFSET]['status']; - if (($commitOffsetStatus & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - } - } - - // }}} - // {{{ protected function processing() - - protected function processing($key, $context) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - // set process start time - $this->callStatus[$key]['time'] = microtime(true); - switch ($key) { - case self::REQUEST_METADATA: - case self::REQUEST_GETGROUP: - case self::REQUEST_JOINGROUP: - case self::REQUEST_SYNCGROUP: - case self::REQUEST_HEARTGROUP: - case self::REQUEST_FETCH_OFFSET: - case self::REQUEST_COMMIT_OFFSET: - $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; - break; - case self::REQUEST_OFFSET: - case self::REQUEST_FETCH: - $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; - $contextStatus = array(); - foreach ($context as $fd) { - $contextStatus[$fd] = self::STATUS_PROCESS; - } - $this->callStatus[$key]['context'] = $contextStatus; - break; - - } - } - - // }}} - // {{{ protected function report() - - protected function report() - { - //var_dump($this->callStatus[self::REQUEST_COMMIT_OFFSET]); - } - - // }}} - // }}} -} diff --git a/Kafka/ConsumerConfig.php b/Kafka/ConsumerConfig.php deleted file mode 100644 index b983f225..00000000 --- a/Kafka/ConsumerConfig.php +++ /dev/null @@ -1,149 +0,0 @@ - '', - 'sessionTimeout' => 30000, - 'rebalanceTimeout' => 30000, - 'topics' => array(), - 'offsetReset' => 'latest', // earliest - 'maxBytes' => 65536, // 64kb - 'maxWaitTime' => 100, - ); - - // }}} - // {{{ functions - // {{{ public function getGroupId() - - public function getGroupId() - { - $groupId = trim($this->ietGroupId()); - - if ($groupId == false || $groupId == '') { - throw new \Kafka\Exception\Config('Get group id value is invalid, must set it not empty string'); - } - - return $groupId; - } - - // }}} - // {{{ public function setGroupId() - - public function setGroupId($groupId) - { - $groupId = trim($groupId); - if ($groupId == false || $groupId == '') { - throw new \Kafka\Exception\Config('Set group id value is invalid, must set it not empty string'); - } - static::$options['groupId'] = $groupId; - } - - // }}} - // {{{ public function setSessionTimeout() - - public function setSessionTimeout($sessionTimeout) - { - if (!is_numeric($sessionTimeout) || $sessionTimeout < 1 || $sessionTimeout > 3600000) { - throw new \Kafka\Exception\Config('Set session timeout value is invalid, must set it 1 .. 3600000'); - } - static::$options['sessionTimeout'] = $sessionTimeout; - } - - // }}} - // {{{ public function setRebalanceTimeout() - - public function setRebalanceTimeout($rebalanceTimeout) - { - if (!is_numeric($rebalanceTimeout) || $rebalanceTimeout < 1 || $rebalanceTimeout > 3600000) { - throw new \Kafka\Exception\Config('Set rebalance timeout value is invalid, must set it 1 .. 3600000'); - } - static::$options['rebalanceTimeout'] = $rebalanceTimeout; - } - - // }}} - // {{{ public function setOffsetReset() - - public function setOffsetReset($offsetReset) - { - if (!in_array($offsetReset, array('latest', 'earliest'))) { - throw new \Kafka\Exception\Config('Set offset reset value is invalid, must set it `latest` or `earliest`'); - } - static::$options['offsetReset'] = $offsetReset; - } - - // }}} - // {{{ public function getTopics() - - public function getTopics() - { - $topics = $this->ietTopics(); - - if (empty($topics)) { - throw new \Kafka\Exception\Config('Get consumer topics value is invalid, must set it not empty'); - } - - return $topics; - } - - // }}} - // {{{ public function setTopics() - - public function setTopics($topics) - { - if (!is_array($topics) || empty($topics)) { - throw new \Kafka\Exception\Config('Set consumer topics value is invalid, must set it not empty array'); - } - static::$options['topics'] = $topics; - } - - // }}} - // }}} - - protected $runtime_options = [ - 'consume_mode' => self::CONSUME_AFTER_COMMIT_OFFSET - ]; - const CONSUME_AFTER_COMMIT_OFFSET = 1; - const CONSUME_BEFORE_COMMIT_OFFSET = 2; - - public function setConsumeMode($mode) - { - $this->runtime_options['consume_mode'] = $mode; - } - - public function getConsumeMode() - { - return $this->runtime_options['consume_mode']; - } -} diff --git a/Kafka/Exception.php b/Kafka/Exception.php deleted file mode 100644 index f336f1c3..00000000 --- a/Kafka/Exception.php +++ /dev/null @@ -1,31 +0,0 @@ -get('server'); + $application->set('kafka', [ + 'class' => Producer::class + ]); $kafka = SConfig::get('kafka'); if (empty($kafka) || !($kafka['enable'] ?? false)) { diff --git a/Kafka/Producer.php b/Kafka/Producer.php index f290bd12..113e2cd6 100644 --- a/Kafka/Producer.php +++ b/Kafka/Producer.php @@ -14,116 +14,49 @@ namespace Kafka; +use RdKafka\Conf; +use RdKafka\TopicConf; +use Snowflake\Abstracts\Component; + /** -+------------------------------------------------------------------------------ -* Kafka protocol since Kafka v0.8 -+------------------------------------------------------------------------------ -* -* @package -* @version $_SWANBR_VERSION_$ -* @copyright Copyleft -* @author $_SWANBR_AUTHOR_$ -+------------------------------------------------------------------------------ -*/ - -class Producer + * +------------------------------------------------------------------------------ + * Kafka protocol since Kafka v0.8 + * +------------------------------------------------------------------------------ + * + * @package + * @version $_SWANBR_VERSION_$ + * @copyright Copyleft + * @author $_SWANBR_AUTHOR_$ + * +------------------------------------------------------------------------------ + */ +class Producer extends Component { - use \Psr\Log\LoggerAwareTrait; - use \Kafka\LoggerTrait; - // {{{ consts - // }}} - // {{{ members - - private $process = null; + /** + * @param $topic + * @param $brokers + * @param $message + * @param null $key + * @param int $timeout + */ + public function delivery($topic, $brokers, $message, $key = null, $timeout = 5) + { + $conf = new Conf(); + $conf->set('metadata.broker.list', $brokers); + $conf->setDrmSgCb(function ($kafka, $message) { +// $this->debug(var_export($message, true)); + }); + $conf->setErrorCb(function ($kafka, $err, $reason) { + $this->error(sprintf("Kafka error: %s (reason: %s)", rd_kafka_err2str($err), $reason)); + }); - // }}} - // {{{ functions - // {{{ public function __construct() + $cf = new TopicConf(); + $cf->set('request.required.acks', 1); - /** - * __construct - * - * @access public - * @param $hostList - * @param null $timeout - */ - public function __construct(\Closure $producer = null) - { - if (is_null($producer)) { - $this->process = new \Kafka\Producer\SyncProcess(); - } else { - $this->process = new \Kafka\Producer\Process($producer); - } - } - - // }}} - // {{{ public function send() - - /** - * start producer - * - * @access public - * @data is data is boolean that is async process, thus it is sync process - * @return void - */ - public function send($data = true) - { - if ($this->logger) { - $this->process->setLogger($this->logger); - } - if (is_bool($data)) { - $this->process->start(); - if ($data) { - \Amp\run(); - } - } else { - return $this->process->send($data); - } - } - - // }}} - // {{{ public function syncMeta() - - /** - * syncMeta producer - * - * @access public - * @return void - */ - public function syncMeta() - { - return $this->process->syncMeta(); - } - - // }}} - // {{{ public function success() - - /** - * producer success - * - * @access public - * @return void - */ - public function success(\Closure $success = null) - { - $this->process->setSuccess($success); - } - - // }}} - // {{{ public function error() - - /** - * producer error - * - * @access public - * @return void - */ - public function error(\Closure $error = null) - { - $this->process->setError($error); - } - - // }}} - // }}} + $rk = new \RdKafka\Producer($conf); + $topic = $rk->newTopic($topic, $cf); + $topic->produce(RD_KAFKA_PARTITION_UA, 0, $message, $key); + $rk->poll($timeout); + $rk->flush($timeout); + } } diff --git a/Kafka/Producer/Process.php b/Kafka/Producer/Process.php deleted file mode 100644 index 589db2cd..00000000 --- a/Kafka/Producer/Process.php +++ /dev/null @@ -1,384 +0,0 @@ -producer = $producer; - } - - // }}} - // {{{ public function init() - - /** - * start consumer - * - * @access public - * @return void - */ - public function init() - { - // init protocol - $config = \Kafka\ProducerConfig::getInstance(); - \Kafka\Protocol::init($config->getBrokerVersion(), $this->logger); - - // init process request - $broker = \Kafka\Broker::getInstance(); - $broker->setProcess(function ($data, $fd) { - $this->processRequest($data, $fd); - }); - - // init state - $this->state = \Kafka\Producer\State::getInstance(); - if ($this->logger) { - $this->state->setLogger($this->logger); - } - $this->state->setCallback(array( - \Kafka\Producer\State::REQUEST_METADATA => function () { - return $this->syncMeta(); - }, - \Kafka\Producer\State::REQUEST_PRODUCE => function () { - return $this->produce(); - }, - )); - $this->state->init(); - - $topics = $broker->getTopics(); - if (!empty($topics)) { - $this->state->succRun(\Kafka\Producer\State::REQUEST_METADATA); - } - } - - // }}} - // {{{ public function start() - - /** - * start consumer - * - * @access public - * @return void - */ - public function start() - { - $this->init(); - $this->state->start(); - $config = \Kafka\ProducerConfig::getInstance(); - $isAsyn = $config->getIsAsyn(); - if (!$isAsyn) { - \Amp\repeat(function ($watcherId) { - if ($this->error) { - call_user_func($this->error, 1000); - } - \Amp\cancel($watcherId); - \Amp\stop(); - }, $msInterval = $config->getRequestTimeout()); - }; - } - - // }}} - // {{{ public function stop() - - /** - * stop consumer - * - * @access public - * @return void - */ - public function stop() - { - $this->isRunning = false; - } - - // }}} - // {{{ public function setSuccess() - - /** - * set success callback - * - * @access public - * @return void - */ - public function setSuccess($success) - { - $this->success = $success; - } - - // }}} - // {{{ public function setError() - - /** - * set error callback - * - * @access public - * @return void - */ - public function setError($error) - { - $this->error = $error; - } - - // }}} - // {{{ public function syncMeta() - - public function syncMeta() - { - $this->debug('Start sync metadata request'); - $brokerList = explode(',', \Kafka\ProducerConfig::getInstance()->getMetadataBrokerList()); - $brokerHost = array(); - foreach ($brokerList as $key => $val) { - if (trim($val)) { - $brokerHost[] = $val; - } - } - if (count($brokerHost) == 0) { - throw new \Kafka\Exception('Not set config `metadataBrokerList`'); - } - shuffle($brokerHost); - $broker = \Kafka\Broker::getInstance(); - foreach ($brokerHost as $host) { - $socket = $broker->getMetaConnect($host); - if ($socket) { - $params = array(); - $this->debug('Start sync metadata request params:' . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); - $socket->write($requestData); - return; - } - } - throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); - } - - // }}} - // {{{ protected function processRequest() - - /** - * process Request - * - * @access public - * @return void - */ - protected function processRequest($data, $fd) - { - $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); - switch ($correlationId) { - case \Kafka\Protocol::METADATA_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); - if (!isset($result['brokers']) || !isset($result['topics'])) { - $this->error('Get metadata is fail, brokers or topics is null.'); - $this->state->failRun(\Kafka\Producer\State::REQUEST_METADATA); - } else { - $broker = \Kafka\Broker::getInstance(); - $isChange = $broker->setData($result['topics'], $result['brokers']); - $this->state->succRun(\Kafka\Producer\State::REQUEST_METADATA, $isChange); - } - break; - case \Kafka\Protocol::PRODUCE_REQUEST: - $result = \Kafka\Protocol::decode(\Kafka\Protocol::PRODUCE_REQUEST, substr($data, 4)); - $this->succProduce($result, $fd); - break; - default: - $this->error('Error request, correlationId:' . $correlationId); - } - } - - // }}} - // {{{ protected function produce() - - protected function produce() - { - $context = array(); - $broker = \Kafka\Broker::getInstance(); - $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); - $timeout = \Kafka\ProducerConfig::getInstance()->getTimeout(); - - // get send message - // data struct - // topic: - // partId: - // key: - // value: - $data = call_user_func($this->producer); - if (empty($data)) { - return $context; - } - - $sendData = $this->convertMessage($data); - foreach ($sendData as $brokerId => $topicList) { - $connect = $broker->getDataConnect($brokerId); - if (!$connect) { - return; - } - - $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); - $params = array( - 'required_ack' => $requiredAck, - 'timeout' => \Kafka\ProducerConfig::getInstance()->getTimeout(), - 'data' => $topicList, - ); - $this->debug("Send message start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::PRODUCE_REQUEST, $params); - if ($requiredAck == 0) { // If it is 0 the server will not send any response - $this->state->succRun(\Kafka\Producer\State::REQUEST_PRODUCE); - } else { - $connect->write($requestData); - $context[] = (int)$connect->getSocket(); - } - } - - return $context; - } - - // }}} - // {{{ protected function succProduce() - - protected function succProduce($result, $fd) - { - $msg = sprintf('Send message sucess, result: %s', json_encode($result)); - $this->debug($msg); - if ($this->success) { - call_user_func($this->success, $result); - } - $this->state->succRun(\Kafka\Producer\State::REQUEST_PRODUCE, $fd); - } - - // }}} - // {{{ protected function stateConvert() - - protected function stateConvert($errorCode, $context = null) - { - $retry = false; - $this->error(\Kafka\Protocol::getError($errorCode)); - if ($this->error) { - call_user_func($this->error, $errorCode); - } - $recoverCodes = array( - \Kafka\Protocol::UNKNOWN_TOPIC_OR_PARTITION, - \Kafka\Protocol::INVALID_REQUIRED_ACKS, - \Kafka\Protocol::RECORD_LIST_TOO_LARGE, - \Kafka\Protocol::NOT_ENOUGH_REPLICAS_AFTER_APPEND, - \Kafka\Protocol::NOT_ENOUGH_REPLICAS, - \Kafka\Protocol::NOT_LEADER_FOR_PARTITION, - \Kafka\Protocol::BROKER_NOT_AVAILABLE, - \Kafka\Protocol::GROUP_LOAD_IN_PROGRESS, - \Kafka\Protocol::GROUP_COORDINATOR_NOT_AVAILABLE, - \Kafka\Protocol::NOT_COORDINATOR_FOR_GROUP, - \Kafka\Protocol::INVALID_TOPIC, - \Kafka\Protocol::INCONSISTENT_GROUP_PROTOCOL, - \Kafka\Protocol::INVALID_GROUP_ID, - ); - if (in_array($errorCode, $recoverCodes)) { - $this->state->recover(); - return false; - } - return true; - } - - // }}} - // {{{ protected function convertMessage() - - protected function convertMessage($data) - { - $sendData = array(); - $broker = \Kafka\Broker::getInstance(); - $topicInfos = $broker->getTopics(); - foreach ($data as $value) { - if (!isset($value['topic']) || !trim($value['topic'])) { - continue; - } - - if (!isset($topicInfos[$value['topic']])) { - continue; - } - - if (!isset($value['value']) || !trim($value['value'])) { - continue; - } - - if (!isset($value['key'])) { - $value['key'] = ''; - } - - $topicMeta = $topicInfos[$value['topic']]; - $partNums = array_keys($topicMeta); - shuffle($partNums); - $partId = 0; - if (!isset($value['partId']) || !isset($topicMeta[$value['partId']])) { - $partId = $partNums[0]; - } else { - $partId = $value['partId']; - } - - $brokerId = $topicMeta[$partId]; - $topicData = array(); - if (isset($sendData[$brokerId][$value['topic']])) { - $topicData = $sendData[$brokerId][$value['topic']]; - } - - $partition = array(); - if (isset($topicData['partitions'][$partId])) { - $partition = $topicData['partitions'][$partId]; - } - - $partition['partition_id'] = $partId; - if (trim($value['key']) != '') { - $partition['messages'][] = array('value' => $value['value'], 'key' => $value['key']); - } else { - $partition['messages'][] = $value['value']; - } - - $topicData['partitions'][$partId] = $partition; - $topicData['topic_name'] = $value['topic']; - $sendData[$brokerId][$value['topic']] = $topicData; - } - - return $sendData; - } - - // }}} - // }}} -} diff --git a/Kafka/Producer/State.php b/Kafka/Producer/State.php deleted file mode 100644 index 1984d895..00000000 --- a/Kafka/Producer/State.php +++ /dev/null @@ -1,275 +0,0 @@ - array(), - self::REQUEST_PRODUCE => array(), - ); - - // }}} - // {{{ functions - // {{{ public function init() - - public function init() - { - $this->callStatus = array( - self::REQUEST_METADATA => array( - 'status'=> self::STATUS_LOOP, - ), - self::REQUEST_PRODUCE => array( - 'status'=> self::STATUS_LOOP, - ), - ); - - // instances clear - - // init requests - $config = \Kafka\ConsumerConfig::getInstance(); - foreach ($this->requests as $request => $option) { - switch ($request) { - case self::REQUEST_METADATA: - $this->requests[$request]['interval'] = $config->getMetadataRefreshIntervalMs(); - break; - default: - $isAsyn = $config->getIsAsyn(); - if ($isAsyn) { - $this->requests[$request]['interval'] = $config->getProduceInterval(); - } else { - $this->requests[$request]['interval'] = 1; - } - } - } - } - - // }}} - // {{{ public function start() - - public function start() - { - foreach ($this->requests as $request => $option) { - $interval = isset($option['interval']) ? $option['interval'] : 200; - \Amp\repeat(function ($watcherId) use ($request, $option) { - if ($this->checkRun($request) && $option['func'] != null) { - $context = call_user_func($option['func']); - $this->processing($request, $context); - } - $this->requests[$request]['watcher'] = $watcherId; - }, $msInterval = $interval); - } - - // start sync metadata - if (isset($this->requests[self::REQUEST_METADATA]['func']) - && $this->callStatus[self::REQUEST_METADATA]['status'] == self::STATUS_LOOP) { - $context = call_user_func($this->requests[self::REQUEST_METADATA]['func']); - $this->processing($request, $context); - } - \Amp\repeat(function ($watcherId) { - $this->report(); - }, $msInterval = 1000); - } - - // }}} - // {{{ public function succRun() - - public function succRun($key, $context = null) - { - $config = \Kafka\ConsumerConfig::getInstance(); - $isAsyn = $config->getIsAsyn(); - if (!isset($this->callStatus[$key])) { - return false; - } - - switch ($key) { - case self::REQUEST_METADATA: - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - if ($context) { // if kafka broker is change - $this->recover(); - } - break; - case self::REQUEST_PRODUCE: - if ($context == null) { - if (!$isAsyn) { - $this->callStatus[$key]['status'] = self::STATUS_FINISH; - \Amp\stop(); - } else { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - } - break; - } - unset($this->callStatus[$key]['context'][$context]); - $contextStatus = $this->callStatus[$key]['context']; - if (empty($contextStatus)) { - if (!$isAsyn) { - $this->callStatus[$key]['status'] = self::STATUS_FINISH; - \Amp\stop(); - } else { - $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH); - } - } - break; - } - } - - // }}} - // {{{ public function failRun() - - public function failRun($key, $context = null) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - switch ($key) { - case self::REQUEST_METADATA: - $this->callStatus[$key]['status'] = self::STATUS_LOOP; - break; - case self::REQUEST_PRODUCE: - $this->recover(); - break; - } - } - - // }}} - // {{{ public function setCallback() - - public function setCallback($callbacks) - { - foreach ($callbacks as $request => $callback) { - $this->requests[$request]['func'] = $callback; - } - } - - // }}} - // {{{ public function recover() - - public function recover() - { - $this->callStatus = array( - self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA], - self::REQUEST_PRODUCE => array( - 'status'=> self::STATUS_LOOP, - ), - ); - } - - // }}} - // {{{ protected function checkRun() - - protected function checkRun($key) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - $status = $this->callStatus[$key]['status']; - switch ($key) { - case self::REQUEST_METADATA: - if ($status & self::STATUS_PROCESS == self::STATUS_PROCESS) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - case self::REQUEST_PRODUCE: - if (($status & self::STATUS_PROCESS) == self::STATUS_PROCESS) { - return false; - } - $syncStatus = $this->callStatus[self::REQUEST_METADATA]['status']; - if (($syncStatus & self::STATUS_FINISH) != self::STATUS_FINISH) { - return false; - } - if (($status & self::STATUS_LOOP) == self::STATUS_LOOP) { - return true; - } - return false; - } - } - - // }}} - // {{{ protected function processing() - - protected function processing($key, $context) - { - if (!isset($this->callStatus[$key])) { - return false; - } - - // set process start time - $this->callStatus[$key]['time'] = microtime(true); - switch ($key) { - case self::REQUEST_METADATA: - $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; - break; - case self::REQUEST_PRODUCE: - if (empty($context)) { - break; - } - $this->callStatus[$key]['status'] |= self::STATUS_PROCESS; - $contextStatus = array(); - if (is_array($context)) { - foreach ($context as $fd) { - $contextStatus[$fd] = self::STATUS_PROCESS; - } - $this->callStatus[$key]['context'] = $contextStatus; - } - break; - } - } - - // }}} - // {{{ protected function report() - - protected function report() - { - //var_dump($this->callStatus[self::REQUEST_COMMIT_OFFSET]); - } - - // }}} - // }}} -} diff --git a/Kafka/Producer/SyncProcess.php b/Kafka/Producer/SyncProcess.php deleted file mode 100644 index 3c7f142f..00000000 --- a/Kafka/Producer/SyncProcess.php +++ /dev/null @@ -1,198 +0,0 @@ -getBrokerVersion(), $this->logger); - $this->syncMeta(); - } - - // }}} - // {{{ public function send() - - public function send($data) - { - $broker = \Kafka\Broker::getInstance(); - $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); - $timeout = \Kafka\ProducerConfig::getInstance()->getTimeout(); - - // get send message - // data struct - // topic: - // partId: - // key: - // value: - if (empty($data)) { - return false; - } - - $sendData = $this->convertMessage($data); - $result = array(); - foreach ($sendData as $brokerId => $topicList) { - $connect = $broker->getDataConnect($brokerId, true); - if (!$connect) { - return false; - } - - $requiredAck = \Kafka\ProducerConfig::getInstance()->getRequiredAck(); - $params = array( - 'required_ack' => $requiredAck, - 'timeout' => \Kafka\ProducerConfig::getInstance()->getTimeout(), - 'data' => $topicList, - ); - $this->debug("Send message start, params:" . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::PRODUCE_REQUEST, $params); - $connect->write($requestData); - if ($requiredAck != 0) { // If it is 0 the server will not send any response - $dataLen = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, $connect->read(4)); - $data = $connect->read($dataLen); - $correlationId = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4)); - $ret = \Kafka\Protocol::decode(\Kafka\Protocol::PRODUCE_REQUEST, substr($data, 4)); - $result[] = $ret; - } - } - return $result; - } - - // }}} - // {{{ public function syncMeta() - - public function syncMeta() - { - $this->debug('Start sync metadata request'); - $brokerList = explode(',', \Kafka\ProducerConfig::getInstance()->getMetadataBrokerList()); - $brokerHost = array(); - foreach ($brokerList as $key => $val) { - if (trim($val)) { - $brokerHost[] = $val; - } - } - if (count($brokerHost) == 0) { - throw new \Kafka\Exception('Not set config `metadataBrokerList`'); - } - shuffle($brokerHost); - $broker = \Kafka\Broker::getInstance(); - foreach ($brokerHost as $host) { - $socket = $broker->getMetaConnect($host, true); - if ($socket) { - $params = array(); - $this->debug('Start sync metadata request params:' . json_encode($params)); - $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params); - $socket->write($requestData); - $data = $socket->read(\Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, $socket->read(4))); - $result = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4)); - if (!isset($result['brokers']) || !isset($result['topics'])) { - throw new \Kafka\Exception('Get metadata is fail, brokers or topics is null.'); - } else { - $broker = \Kafka\Broker::getInstance(); - $broker->setData($result['topics'], $result['brokers']); - } - return; - } - } - throw new \Kafka\Exception('Not has broker can connection `metadataBrokerList`'); - } - - // }}} - // {{{ protected function convertMessage() - - protected function convertMessage($data) - { - $sendData = array(); - $broker = \Kafka\Broker::getInstance(); - $topicInfos = $broker->getTopics(); - foreach ($data as $value) { - if (!isset($value['topic']) || !trim($value['topic'])) { - continue; - } - - if (!isset($topicInfos[$value['topic']])) { - continue; - } - - if (!isset($value['value']) || !trim($value['value'])) { - continue; - } - - if (!isset($value['key'])) { - $value['key'] = ''; - } - - $topicMeta = $topicInfos[$value['topic']]; - $partNums = array_keys($topicMeta); - shuffle($partNums); - $partId = 0; - if (!isset($value['partId']) || !isset($topicMeta[$value['partId']])) { - $partId = $partNums[0]; - } else { - $partId = $value['partId']; - } - - $brokerId = $topicMeta[$partId]; - $topicData = array(); - if (isset($sendData[$brokerId][$value['topic']])) { - $topicData = $sendData[$brokerId][$value['topic']]; - } - - $partition = array(); - if (isset($topicData['partitions'][$partId])) { - $partition = $topicData['partitions'][$partId]; - } - - $partition['partition_id'] = $partId; - if (trim($value['key']) != '') { - $partition['messages'][] = array('value' => $value['value'], 'key' => $value['key']); - } else { - $partition['messages'][] = $value['value']; - } - - $topicData['partitions'][$partId] = $partition; - $topicData['topic_name'] = $value['topic']; - $sendData[$brokerId][$value['topic']] = $topicData; - } - - return $sendData; - } - - // }}} - // }}} -} diff --git a/Kafka/ProducerConfig.php b/Kafka/ProducerConfig.php deleted file mode 100644 index 8ff171bb..00000000 --- a/Kafka/ProducerConfig.php +++ /dev/null @@ -1,102 +0,0 @@ - 1, - 'timeout' => 5000, - 'isAsyn' => false, - 'requestTimeout' => 6000, - 'produceInterval' => 100, - ); - - // }}} - // {{{ functions - // {{{ public function setRequestTimeout() - - public function setRequestTimeout($requestTimeout) - { - if (!is_numeric($requestTimeout) || $requestTimeout < 1 || $requestTimeout > 900000) { - throw new \Kafka\Exception\Config('Set request timeout value is invalid, must set it 1 .. 900000'); - } - static::$options['requestTimeout'] = $requestTimeout; - } - - // }}} - // {{{ public function setProduceInterval() - - public function setProduceInterval($produceInterval) - { - if (!is_numeric($produceInterval) || $produceInterval < 1 || $produceInterval > 900000) { - throw new \Kafka\Exception\Config('Set produce interval timeout value is invalid, must set it 1 .. 900000'); - } - static::$options['produceInterval'] = $produceInterval; - } - - // }}} - // {{{ public function setTimeout() - - public function setTimeout($timeout) - { - if (!is_numeric($timeout) || $timeout < 1 || $timeout > 900000) { - throw new \Kafka\Exception\Config('Set timeout value is invalid, must set it 1 .. 900000'); - } - static::$options['timeout'] = $timeout; - } - - // }}} - // {{{ public function setRequiredAck() - - public function setRequiredAck($requiredAck) - { - if (!is_numeric($requiredAck) || $requiredAck < -1 || $requiredAck > 1000) { - throw new \Kafka\Exception\Config('Set required ack value is invalid, must set it -1 .. 1000'); - } - static::$options['requiredAck'] = $requiredAck; - } - - // }}} - // {{{ public function setIsAsyn() - - public function setIsAsyn($asyn) - { - if (!is_bool($asyn)) { - throw new \Kafka\Exception\Config('Set isAsyn value is invalid, must set it bool value'); - } - static::$options['isAsyn'] = $asyn; - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol.php b/Kafka/Protocol.php deleted file mode 100644 index 9fa5706a..00000000 --- a/Kafka/Protocol.php +++ /dev/null @@ -1,294 +0,0 @@ - 'Produce', - \Kafka\Protocol\Protocol::FETCH_REQUEST => 'Fetch', - \Kafka\Protocol\Protocol::OFFSET_REQUEST => 'Offset', - \Kafka\Protocol\Protocol::METADATA_REQUEST => 'Metadata', - \Kafka\Protocol\Protocol::OFFSET_COMMIT_REQUEST => 'CommitOffset', - \Kafka\Protocol\Protocol::OFFSET_FETCH_REQUEST => 'FetchOffset', - \Kafka\Protocol\Protocol::GROUP_COORDINATOR_REQUEST => 'GroupCoordinator', - \Kafka\Protocol\Protocol::JOIN_GROUP_REQUEST => 'JoinGroup', - \Kafka\Protocol\Protocol::HEART_BEAT_REQUEST => 'Heartbeat', - \Kafka\Protocol\Protocol::LEAVE_GROUP_REQUEST => 'LeaveGroup', - \Kafka\Protocol\Protocol::SYNC_GROUP_REQUEST => 'SyncGroup', - \Kafka\Protocol\Protocol::DESCRIBE_GROUPS_REQUEST => 'DescribeGroups', - \Kafka\Protocol\Protocol::LIST_GROUPS_REQUEST => 'ListGroup', - ); - - $namespace = '\\Kafka\\Protocol\\'; - foreach ($class as $key => $className) { - $class = $namespace . $className; - self::$objects[$key] = new $class($version); - if ($logger) { - self::$objects[$key]->setLogger($logger); - } - } - } - - // }}} - // {{{ public static function encode() - - /** - * request encode - * - * @param key $appkey - * @param array $payloads - * @access public - * @return string - */ - public static function encode($key, $payloads) - { - if (!isset(self::$objects[$key])) { - throw new \Kafka\Exception('Not support api key, key:' . $key); - } - - return self::$objects[$key]->encode($payloads); - } - - // }}} - // {{{ public static function decode() - - /** - * decode response - * - * @access public - * @return array - */ - public static function decode($key, $data) - { - if (!isset(self::$objects[$key])) { - throw new \Kafka\Exception('Not support api key, key:' . $key); - } - - return self::$objects[$key]->decode($data); - } - - // }}} - // {{{ public static function getError() - - /** - * get error - * - * @param integer $errCode - * @static - * @access public - * @return string - */ - public static function getError($errCode) - { - switch ($errCode) { - case 0: - $error = 'No error--it worked!'; - break; - case -1: - $error = 'An unexpected server error'; - break; - case 1: - $error = 'The requested offset is outside the range of offsets maintained by the server for the given topic/partition.'; - break; - case 2: - $error = 'This indicates that a message contents does not match its CRC'; - break; - case 3: - $error = 'This request is for a topic or partition that does not exist on this broker.'; - break; - case 4: - $error = 'The message has a negative size'; - break; - case 5: - $error = 'This error is thrown if we are in the middle of a leadership election and there is currently no leader for this partition and hence it is unavailable for writes'; - break; - case 6: - $error = 'This error is thrown if the client attempts to send messages to a replica that is not the leader for some partition. It indicates that the clients metadata is out of date.'; - break; - case 7: - $error = 'This error is thrown if the request exceeds the user-specified time limit in the request.'; - break; - case 8: - $error = 'This is not a client facing error and is used only internally by intra-cluster broker communication.'; - break; - case 9: - $error = 'The replica is not available for the requested topic-partition'; - break; - case 10: - $error = 'The server has a configurable maximum message size to avoid unbounded memory allocation. This error is thrown if the client attempt to produce a message larger than this maximum.'; - break; - case 11: - $error = 'Internal error code for broker-to-broker communication.'; - break; - case 12: - $error = 'If you specify a string larger than configured maximum for offset metadata'; - break; - case 13: - $error = 'The server disconnected before a response was received.'; - break; - case 14: - $error = 'The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition).'; - break; - case 15: - $error = 'The broker returns this error code for consumer metadata requests or offset commit requests if the offsets topic has not yet been created.'; - break; - case 16: - $error = 'The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for.'; - break; - case 17: - $error = 'The request attempted to perform an operation on an invalid topic.'; - break; - case 18: - $error = 'The request included message batch larger than the configured segment size on the server.'; - break; - case 19: - $error = 'Messages are rejected since there are fewer in-sync replicas than required.'; - break; - case 20: - $error = 'Messages are written to the log, but to fewer in-sync replicas than required.'; - break; - case 21: - $error = 'Produce request specified an invalid value for required acks.'; - break; - case 22: - $error = 'Specified group generation id is not valid.'; - break; - case 23: - $error = 'The group member\'s supported protocols are incompatible with those of existing members.'; - break; - case 24: - $error = 'The configured groupId is invalid'; - break; - case 25: - $error = 'The coordinator is not aware of this member.'; - break; - case 26: - $error = 'The session timeout is not within the range allowed by the broker (as configured by group.min.session.timeout.ms and group.max.session.timeout.ms).'; - break; - case 27: - $error = 'The group is rebalancing, so a rejoin is needed.'; - break; - case 28: - $error = 'The committing offset data size is not valid'; - break; - case 29: - $error = 'Topic authorization failed.'; - break; - case 30: - $error = 'Group authorization failed.'; - break; - case 31: - $error = 'Cluster authorization failed.'; - break; - case 32: - $error = 'The timestamp of the message is out of acceptable range.'; - break; - case 33: - $error = 'The broker does not support the requested SASL mechanism.'; - break; - case 34: - $error = 'Request is not valid given the current SASL state.'; - break; - case 35: - $error = 'The version of API is not supported.'; - break; - default: - $error = 'Unknown error'; - } - - return $error; - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/CommitOffset.php b/Kafka/Protocol/CommitOffset.php deleted file mode 100644 index fc84a077..00000000 --- a/Kafka/Protocol/CommitOffset.php +++ /dev/null @@ -1,226 +0,0 @@ -getApiVersion(self::OFFSET_COMMIT_REQUEST); - - $header = $this->requestHeader('kafka-php', self::OFFSET_COMMIT_REQUEST, self::OFFSET_COMMIT_REQUEST); - - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - if ($version == self::API_VERSION1) { - $data .= self::pack(self::BIT_B32, $payloads['generation_id']); - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - } - if ($version == self::API_VERSION2) { - $data .= self::pack(self::BIT_B32, $payloads['generation_id']); - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - $data .= self::pack(self::BIT_B64, $payloads['retention_time']); - } - - $data .= self::encodeArray($payloads['data'], array($this, 'encodeTopic')); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $version = $this->getApiVersion(self::OFFSET_REQUEST); - $topics = $this->decodeArray(substr($data, $offset), array($this, 'decodeTopic'), $version); - $offset += $topics['length']; - - return $topics['data']; - } - - // }}} - // {{{ protected function encodeTopic() - - /** - * encode commit offset topic array - * - * @param array $values - * @access public - * @return array - */ - protected function encodeTopic($values) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given commit offset data invalid. `topic_name` is undefined.'); - } - if (!isset($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given commit offset data invalid. `partitions` is undefined.'); - } - - $data = self::encodeString($values['topic_name'], self::PACK_INT16); - $data .= self::encodeArray($values['partitions'], array($this, 'encodePartition')); - - return $data; - } - - // }}} - // {{{ protected function encodePartition() - - /** - * encode commit offset partition array - * - * @param array $values - * @access public - * @return array - */ - protected function encodePartition($values) - { - if (!isset($values['partition'])) { - throw new \Kafka\Exception\Protocol('given commit offset data invalid. `partition` is undefined.'); - } - if (!isset($values['offset'])) { - throw new \Kafka\Exception\Protocol('given commit offset data invalid. `offset` is undefined.'); - } - if (!isset($values['metadata'])) { - $values['metadata'] = ''; - } - if (!isset($values['timestamp'])) { - $values['timestamp'] = time() * 1000; - } - $version = $this->getApiVersion(self::OFFSET_COMMIT_REQUEST); - - $data = self::pack(self::BIT_B32, $values['partition']); - $data .= self::pack(self::BIT_B64, $values['offset']); - if ($version == self::API_VERSION1) { - $data .= self::pack(self::BIT_B64, $values['timestamp']); - } - $data .= self::encodeString($values['metadata'], self::PACK_INT16); - - return $data; - } - - // }}} - // {{{ protected function decodeTopic() - - /** - * decode commit offset topic response - * - * @param byte[] $data - * @param string $version - * @access protected - * @return array - */ - protected function decodeTopic($data, $version) - { - $offset = 0; - $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicInfo['length']; - - $partitions = $this->decodeArray(substr($data, $offset), array($this, 'decodePartition'), $version); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicInfo['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // {{{ protected function decodePartition() - - /** - * decode commit offset partition response - * - * @param byte[] $data - * @param string $version - * @access protected - * @return array - */ - protected function decodePartition($data, $version) - { - $offset = 0; - - $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - - - return array( - 'length' => $offset, - 'data' => array( - 'partition' => $partitionId, - 'errorCode' => $errorCode, - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/DescribeGroups.php b/Kafka/Protocol/DescribeGroups.php deleted file mode 100644 index e44814a1..00000000 --- a/Kafka/Protocol/DescribeGroups.php +++ /dev/null @@ -1,201 +0,0 @@ -requestHeader('kafka-php', self::DESCRIBE_GROUPS_REQUEST, self::DESCRIBE_GROUPS_REQUEST); - $data = self::encodeArray($payloads, array($this, 'encodeString'), self::PACK_INT16); - - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $groups = $this->decodeArray(substr($data, $offset), array($this, 'describeGroup')); - $offset += $groups['length']; - - return $groups['data']; - } - - // }}} - // {{{ protected function describeGroup() - - /** - * decode describe group response - * - * @access protected - * @return array - */ - protected function describeGroup($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $groupId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $groupId['length']; - $state = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $state['length']; - $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $protocolType['length']; - $protocol = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $protocol['length']; - - $members = $this->decodeArray(substr($data, $offset), array($this, 'describeMember')); - $offset += $members['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'errorCode' => $errorCode, - 'groupId' => $groupId['data'], - 'state' => $state['data'], - 'protocolType' => $protocolType['data'], - 'protocol' => $protocol['data'], - 'members' => $members['data'] - ) - ); - } - - // }}} - // {{{ protected function describeMember() - - /** - * decode describe members response - * - * @access protected - * @return array - */ - protected function describeMember($data) - { - $offset = 0; - $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $memberId['length']; - $clientId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $clientId['length']; - $clientHost = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $clientHost['length']; - $metadata = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $metadata['length']; - $assignment = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $assignment['length']; - - $memberAssignment = $assignment['data']; - $memberAssignmentOffset = 0; - $version = self::unpack(self::BIT_B16_SIGNED, substr($memberAssignment, $memberAssignmentOffset, 2)); - $memberAssignmentOffset += 2; - $partitionAssignments = $this->decodeArray(substr($memberAssignment, $memberAssignmentOffset), - array($this, 'describeResponsePartition')); - $memberAssignmentOffset += $partitionAssignments['length']; - $userData = $this->decodeString(substr($memberAssignment, $memberAssignmentOffset), self::BIT_B32); - - $metaData = $metadata['data']; - $metaOffset = 0; - $version = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2)); - $metaOffset += 2; - $topics = $this->decodeArray(substr($metaData, $metaOffset), array($this, 'decodeString'), self::BIT_B16); - $metaOffset += $topics['length']; - $metaUserData = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32); - - - return array( - 'length' => $offset, - 'data' => array( - 'memberId' => $memberId['data'], - 'clientId' => $clientId['data'], - 'clientHost' => $clientHost['data'], - 'metadata' => array( - 'version' => $version, - 'topics' => $topics['data'], - 'userData' => $metaUserData['data'], - ), - 'assignment' => array( - 'version' => $version, - 'partitions' => $partitionAssignments['data'], - 'userData' => $userData['data'] - ) - ) - ); - } - - // }}} - // {{{ protected function describeResponsePartition() - - /** - * decode describe group partition response - * - * @access protected - * @return array - */ - protected function describeResponsePartition($data) - { - $offset = 0; - $topicName = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicName['length']; - $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicName['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Fetch.php b/Kafka/Protocol/Fetch.php deleted file mode 100644 index 259a4764..00000000 --- a/Kafka/Protocol/Fetch.php +++ /dev/null @@ -1,377 +0,0 @@ -requestHeader('kafka-php', self::FETCH_REQUEST, self::FETCH_REQUEST); - $data = self::pack(self::BIT_B32, $payloads['replica_id']); - $data .= self::pack(self::BIT_B32, $payloads['max_wait_time']); - $data .= self::pack(self::BIT_B32, $payloads['min_bytes']); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeFetchTopic')); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode fetch response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $version = $this->getApiVersion(self::FETCH_REQUEST); - $throttleTime = 0; - if ($version != self::API_VERSION0) { - $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - } - - $topics = $this->decodeArray(substr($data, $offset), array($this, 'fetchTopic'), $version); - $offset += $topics['length']; - - return array( - 'throttleTime' => $throttleTime, - 'topics' => $topics['data'], - ); - } - - // }}} - // {{{ protected function fetchTopic() - - /** - * decode fetch topic response - * - * @access protected - * @return array - */ - protected function fetchTopic($data, $version) - { - $offset = 0; - $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicInfo['length']; - - $partitions = $this->decodeArray(substr($data, $offset), array($this, 'fetchPartition'), $version); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicInfo['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // {{{ protected function fetchPartition() - - /** - * decode fetch partition response - * - * @access protected - * @return array - */ - protected function fetchPartition($data, $version) - { - $offset = 0; - $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $highwaterMarkOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - - $messageSetSize = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - - if ($offset < strlen($data) && $messageSetSize) { - $messages = $this->decodeMessageSetArray(substr($data, $offset, $messageSetSize), array($this, 'decodeMessageSet'), $messageSetSize); - $offset += $messages['length']; - } - - return array( - 'length' => $offset, - 'data' => array( - 'partition' => $partitionId, - 'errorCode' => $errorCode, - 'highwaterMarkOffset' => $highwaterMarkOffset, - 'messageSetSize' => $messageSetSize, - 'messages' => isset($messages['data']) ? $messages['data'] : array(), - ) - ); - } - - // }}} - // {{{ protected function decodeMessageSetArray() - - /** - * decode message Set - * - * @param array $array - * @param Callable $func - * @param null $options - * @return string - * @access protected - */ - protected function decodeMessageSetArray($data, $func, $messageSetSize = null) - { - $offset = 0; - if (!is_callable($func, false)) { - throw new \Kafka\Exception\Protocol('Decode array failed, given function is not callable.'); - } - - $result = array(); - while ($offset < strlen($data)) { - $value = substr($data, $offset); - if (!is_null($messageSetSize)) { - $ret = call_user_func($func, $value, $messageSetSize); - } else { - $ret = call_user_func($func, $value); - } - - if (!is_array($ret) && $ret === false) { - break; - } - - if (!isset($ret['length']) || !isset($ret['data'])) { - throw new \Kafka\Exception\Protocol('Decode array failed, given function return format is invliad'); - } - if ($ret['length'] == 0) { - continue; - } - - $offset += $ret['length']; - $result[] = $ret['data']; - } - - if ($offset < $messageSetSize) { - $offset = $messageSetSize; - } - - return array('length' => $offset, 'data' => $result); - } - - // }}} - // {{{ protected function decodeMessageSet() - - /** - * decode message set - * N.B., MessageSets are not preceded by an int32 like other array elements - * in the protocol. - * - * @param array $messages - * @param int $compression - * @return string - * @access protected - */ - protected function decodeMessageSet($data) - { - if (strlen($data) <= 12) { - return false; - } - $offset = 0; - $roffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - $messageSize = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $ret = $this->decodeMessage(substr($data, $offset), $messageSize); - if (!is_array($ret) && $ret == false) { - return false; - } - $offset += $ret['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'offset' => $roffset, - 'size' => $messageSize, - 'message' => $ret['data'], - ) - ); - } - - // }}} - // {{{ protected function decodeMessage() - - /** - * decode message - * N.B., MessageSets are not preceded by an int32 like other array elements - * in the protocol. - * - * @param array $messages - * @param int $compression - * @return string - * @access protected - */ - protected function decodeMessage($data, $messageSize) - { - if (strlen($data) < $messageSize || !$messageSize) { - return false; - } - - $offset = 0; - $crc = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $magic = self::unpack(self::BIT_B8, substr($data, $offset, 1)); - $offset += 1; - $attr = self::unpack(self::BIT_B8, substr($data, $offset, 1)); - $offset += 1; - $timestamp = 0; - $backOffset = $offset; - try { // try unpack message format v0 and v1, if use v1 unpack fail, try unpack v0 - $version = $this->getApiVersion(self::FETCH_REQUEST); - if ($version == self::API_VERSION2) { - $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - } - - $keyRet = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $keyRet['length']; - - $valueRet = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $valueRet['length']; - if ($offset != $messageSize) { - throw new \Kafka\Exception('pack message fail, message len:' . $messageSize . ' , data unpack offset :' . $offset); - } - } catch (\Kafka\Exception $e) { // try unpack message format v0 - $offset = $backOffset; - $timestamp = 0; - $keyRet = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $keyRet['length']; - - $valueRet = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $valueRet['length']; - } - - return array( - 'length' => $offset, - 'data' => array( - 'crc' => $crc, - 'magic' => $magic, - 'attr' => $attr, - 'timestamp' => $timestamp, - 'key' => $keyRet['data'], - 'value' => $valueRet['data'], - ) - ); - } - - // }}} - // {{{ protected function encodeFetchPartion() - - /** - * encode signal part - * - * @param partions - * @access protected - * @return string - */ - protected function encodeFetchPartion($values) - { - if (!isset($values['partition_id'])) { - throw new \Kafka\Exception\Protocol('given fetch data invalid. `partition_id` is undefined.'); - } - - if (!isset($values['offset'])) { - $values['offset'] = 0; - } - - if (!isset($values['max_bytes'])) { - $values['max_bytes'] = 2 * 1024 * 1024; - } - - $data = self::pack(self::BIT_B32, $values['partition_id']); - $data .= self::pack(self::BIT_B64, $values['offset']); - $data .= self::pack(self::BIT_B32, $values['max_bytes']); - - return $data; - } - - // }}} - // {{{ protected function encodeFetchTopic() - - /** - * encode signal topic - * - * @param partions - * @access protected - * @return string - */ - protected function encodeFetchTopic($values) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given fetch data invalid. `topic_name` is undefined.'); - } - - if (!isset($values['partitions']) || empty($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given fetch data invalid. `partitions` is undefined.'); - } - - $topic = self::encodeString($values['topic_name'], self::PACK_INT16); - $partitions = self::encodeArray($values['partitions'], array($this, 'encodeFetchPartion')); - - return $topic . $partitions; - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/FetchOffset.php b/Kafka/Protocol/FetchOffset.php deleted file mode 100644 index a0274d61..00000000 --- a/Kafka/Protocol/FetchOffset.php +++ /dev/null @@ -1,185 +0,0 @@ -requestHeader('kafka-php', self::OFFSET_FETCH_REQUEST, self::OFFSET_FETCH_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeOffsetTopic')); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @param byte[] $data - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $version = $this->getApiVersion(self::OFFSET_REQUEST); - $topics = $this->decodeArray(substr($data, $offset), array($this, 'offsetTopic'), $version); - $offset += $topics['length']; - - return $topics['data']; - } - - // }}} - // {{{ protected function encodeOffsetPartion() - - /** - * encode signal part - * - * @param byte[] values partions data - * @access protected - * @return string - */ - protected function encodeOffsetPartion($values) - { - return self::pack(self::BIT_B32, $values); - } - - // }}} - // {{{ protected function encodeOffsetTopic() - - /** - * encode signal topic - * - * @param partions - * @access protected - * @return string - */ - protected function encodeOffsetTopic($values) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given fetch offset data invalid. `topic_name` is undefined.'); - } - - if (!isset($values['partitions']) || empty($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given fetch offset data invalid. `partitions` is undefined.'); - } - - $topic = self::encodeString($values['topic_name'], self::PACK_INT16); - $partitions = self::encodeArray($values['partitions'], array($this, 'encodeOffsetPartion')); - - return $topic . $partitions; - } - - // }}} - // {{{ protected function offsetTopic() - - /** - * decode offset topic response - * - * @access protected - * @return array - */ - protected function offsetTopic($data, $version) - { - $offset = 0; - $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicInfo['length']; - - $partitions = $this->decodeArray(substr($data, $offset), array($this, 'offsetPartition'), $version); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicInfo['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // {{{ protected function offsetPartition() - - /** - * decode offset partition response - * - * @access protected - * @return array - */ - protected function offsetPartition($data, $version) - { - $offset = 0; - - $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - - $roffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - - $metadata = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $metadata['length']; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - - - return array( - 'length' => $offset, - 'data' => array( - 'partition' => $partitionId, - 'errorCode' => $errorCode, - 'metadata' => $metadata['data'], - 'offset' => $roffset, - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/GroupCoordinator.php b/Kafka/Protocol/GroupCoordinator.php deleted file mode 100644 index 13fbd701..00000000 --- a/Kafka/Protocol/GroupCoordinator.php +++ /dev/null @@ -1,85 +0,0 @@ -requestHeader('kafka-php', self::GROUP_COORDINATOR_REQUEST, self::GROUP_COORDINATOR_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $coordinatorId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $hosts = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $hosts['length']; - $coordinatorPort = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - - return array( - 'errorCode' => $errorCode, - 'coordinatorId' => $coordinatorId, - 'coordinatorHost' => $hosts['data'], - 'coordinatorPort' => $coordinatorPort - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Heartbeat.php b/Kafka/Protocol/Heartbeat.php deleted file mode 100644 index f7f84654..00000000 --- a/Kafka/Protocol/Heartbeat.php +++ /dev/null @@ -1,85 +0,0 @@ -requestHeader('kafka-php', self::HEART_BEAT_REQUEST, self::HEART_BEAT_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data .= self::pack(self::BIT_B32, $payloads['generation_id']); - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode heart beat response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - - return array( - 'errorCode' => $errorCode, - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/JoinGroup.php b/Kafka/Protocol/JoinGroup.php deleted file mode 100644 index 0125bd08..00000000 --- a/Kafka/Protocol/JoinGroup.php +++ /dev/null @@ -1,204 +0,0 @@ -requestHeader('kafka-php', self::JOIN_GROUP_REQUEST, self::JOIN_GROUP_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data .= self::pack(self::BIT_B32, $payloads['session_timeout']); - if ($this->getApiVersion(self::JOIN_GROUP_REQUEST) == self::API_VERSION1) { - $data .= self::pack(self::BIT_B32, $payloads['rebalance_timeout']); - } - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - $data .= self::encodeString($payloads['protocol_type'], self::PACK_INT16); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeGroupProtocol')); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode join group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $generationId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $groupProtocol = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $groupProtocol['length']; - $leaderId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $leaderId['length']; - $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $memberId['length']; - - $members = $this->decodeArray(substr($data, $offset), array($this, 'joinGroupMember')); - $offset += $memberId['length']; - - return array( - 'errorCode' => $errorCode, - 'generationId' => $generationId, - 'groupProtocol' => $groupProtocol['data'], - 'leaderId' => $leaderId['data'], - 'memberId' => $memberId['data'], - 'members' => $members['data'], - ); - } - - // }}} - // {{{ protected function encodeGroupProtocol() - - /** - * encode group protocol - * - * @param partions - * @access protected - * @return string - */ - protected function encodeGroupProtocol($values) - { - if (!isset($values['protocol_name'])) { - throw new \Kafka\Exception\Protocol('given join group data invalid. `protocol_name` is undefined.'); - } - - $protocolName = self::encodeString($values['protocol_name'], self::PACK_INT16); - - if (!isset($values['version'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `version` is undefined.'); - } - - if (!isset($values['subscription']) || empty($values['subscription'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `subscription` is undefined.'); - } - if (!isset($values['user_data'])) { - $values['user_data'] = ''; - } - - $data = self::pack(self::BIT_B16, 0); - $data .= self::encodeArray($values['subscription'], array($this, 'encodeGroupProtocolMetaTopic')); - $data .= self::encodeString($values['user_data'], self::PACK_INT32); - - return $protocolName . self::encodeString($data, self::PACK_INT32); - } - - // }}} - // {{{ protected function encodeGroupProtocolMetaTopic() - - /** - * encode group protocol metadata topic - * - * @param partions - * @access protected - * @return string - */ - protected function encodeGroupProtocolMetaTopic($values) - { - $topic = self::encodeString($values, self::PACK_INT16); - return $topic; - } - - // }}} - // {{{ protected function joinGroupMember() - - /** - * decode join group member response - * - * @access protected - * @return array - */ - protected function joinGroupMember($data) - { - $offset = 0; - $memberId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $memberId['length']; - $memberMeta = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $memberMeta['length']; - - $metaData = $memberMeta['data']; - $metaOffset = 0; - $version = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2)); - $metaOffset += 2; - $topics = $this->decodeArray(substr($metaData, $metaOffset), array($this, 'decodeString'), self::BIT_B16); - $metaOffset += $topics['length']; - $userData = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32); - - return array( - 'length' => $offset, - 'data' => array( - 'memberId' => $memberId['data'], - 'memberMeta' => array( - 'version' => $version, - 'topics' => $topics['data'], - 'userData' => $userData['data'], - ), - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/LeaveGroup.php b/Kafka/Protocol/LeaveGroup.php deleted file mode 100644 index c6add5cf..00000000 --- a/Kafka/Protocol/LeaveGroup.php +++ /dev/null @@ -1,79 +0,0 @@ -requestHeader('kafka-php', self::LEAVE_GROUP_REQUEST, self::LEAVE_GROUP_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, 0, 2)); - - return array( - 'errorCode' => $errorCode, - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/ListGroup.php b/Kafka/Protocol/ListGroup.php deleted file mode 100644 index 6f710375..00000000 --- a/Kafka/Protocol/ListGroup.php +++ /dev/null @@ -1,99 +0,0 @@ -requestHeader('kafka-php', self::LIST_GROUPS_REQUEST, self::LIST_GROUPS_REQUEST); - $data = self::encodeString($header, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $groups = $this->decodeArray(substr($data, $offset), array($this, 'listGroup')); - - return array( - 'errorCode' => $errorCode, - 'groups' => $groups['data'], - ); - } - - // }}} - // {{{ protected function listGroup() - - /** - * decode list group response - * - * @access protected - * @return array - */ - protected function listGroup($data) - { - $offset = 0; - $groupId = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $groupId['length']; - $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $protocolType['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'groupId' => $groupId['data'], - 'protocolType' => $protocolType['data'], - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Metadata.php b/Kafka/Protocol/Metadata.php deleted file mode 100644 index 37fde7e9..00000000 --- a/Kafka/Protocol/Metadata.php +++ /dev/null @@ -1,179 +0,0 @@ -requestHeader('kafka-php', self::METADATA_REQUEST, self::METADATA_REQUEST); - $data = self::encodeArray($topics, array($this, 'encodeString'), self::PACK_INT16); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode metadata response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $version = $this->getApiVersion(self::METADATA_REQUEST); - $brokerRet = $this->decodeArray(substr($data, $offset), array($this, 'metaBroker'), $version); - $offset += $brokerRet['length']; - $topicMetaRet = $this->decodeArray(substr($data, $offset), array($this, 'metaTopicMetaData'), $version); - $offset += $topicMetaRet['length']; - - $result = array( - 'brokers' => $brokerRet['data'], - 'topics' => $topicMetaRet['data'], - ); - return $result; - } - - // }}} - // {{{ protected function metaBroker() - - /** - * decode meta broker response - * - * @access protected - * @return array - */ - protected function metaBroker($data, $version) - { - $offset = 0; - $nodeId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $hostNameInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $hostNameInfo['length']; - $port = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - return array( - 'length' => $offset, - 'data' => array( - 'host' => $hostNameInfo['data'], - 'port' => $port, - 'nodeId' => $nodeId - ) - ); - } - - // }}} - // {{{ protected function metaTopicMetaData() - - /** - * decode meta topic meta data response - * - * @access protected - * @return array - */ - protected function metaTopicMetaData($data, $version) - { - $offset = 0; - $topicErrCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicInfo['length']; - $partionsMetaRet = $this->decodeArray(substr($data, $offset), array($this, 'metaPartitionMetaData'), $version); - $offset += $partionsMetaRet['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicInfo['data'], - 'errorCode' => $topicErrCode, - 'partitions' => $partionsMetaRet['data'], - ) - ); - } - - // }}} - // {{{ protected function metaPartitionMetaData() - - /** - * decode meta partition meta data response - * - * @access protected - * @return array - */ - protected function metaPartitionMetaData($data, $version) - { - $offset = 0; - $errcode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $partId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $leader = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $replicas = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); - $offset += $replicas['length']; - $isr = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); - $offset += $isr['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'partitionId' => $partId, - 'errorCode' => $errcode, - 'replicas' => $replicas['data'], - 'leader' => $leader, - 'isr' => $isr['data'], - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Offset.php b/Kafka/Protocol/Offset.php deleted file mode 100644 index 53cb405a..00000000 --- a/Kafka/Protocol/Offset.php +++ /dev/null @@ -1,203 +0,0 @@ -requestHeader('kafka-php', self::OFFSET_REQUEST, self::OFFSET_REQUEST); - $data = self::pack(self::BIT_B32, $payloads['replica_id']); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeOffsetTopic')); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - - $version = $this->getApiVersion(self::OFFSET_REQUEST); - $topics = $this->decodeArray(substr($data, $offset), array($this, 'offsetTopic'), $version); - $offset += $topics['length']; - - return $topics['data']; - } - - // }}} - // {{{ protected function encodeOffsetPartion() - - /** - * encode signal part - * - * @param partions - * @access protected - * @return string - */ - protected function encodeOffsetPartion($values) - { - if (!isset($values['partition_id'])) { - throw new \Kafka\Exception\Protocol('given offset data invalid. `partition_id` is undefined.'); - } - - if (!isset($values['time'])) { - $values['time'] = -1; // -1 - } - - if (!isset($values['max_offset'])) { - $values['max_offset'] = 100000; - } - - $data = self::pack(self::BIT_B32, $values['partition_id']); - $data .= self::pack(self::BIT_B64, $values['time']); - - if ($this->getApiVersion(self::OFFSET_REQUEST) == self::API_VERSION0) { - $data .= self::pack(self::BIT_B32, $values['max_offset']); - } - - return $data; - } - - // }}} - // {{{ protected function encodeOffsetTopic() - - /** - * encode signal topic - * - * @param partions - * @access protected - * @return string - */ - protected function encodeOffsetTopic($values) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given offset data invalid. `topic_name` is undefined.'); - } - - if (!isset($values['partitions']) || empty($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given offset data invalid. `partitions` is undefined.'); - } - - $topic = self::encodeString($values['topic_name'], self::PACK_INT16); - $partitions = self::encodeArray($values['partitions'], array($this, 'encodeOffsetPartion')); - - return $topic . $partitions; - } - - // }}} - // {{{ protected function offsetTopic() - - /** - * decode offset topic response - * - * @access protected - * @return array - */ - protected function offsetTopic($data, $version) - { - $offset = 0; - $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicInfo['length']; - - $partitions = $this->decodeArray(substr($data, $offset), array($this, 'offsetPartition'), $version); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicInfo['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // {{{ protected function offsetPartition() - - /** - * decode offset partition response - * - * @access protected - * @return array - */ - protected function offsetPartition($data, $version) - { - $offset = 0; - $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $timestamp = 0; - if ($version != self::API_VERSION0) { - $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - } - $offsets = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B64); - $offset += $offsets['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'partition' => $partitionId, - 'errorCode' => $errorCode, - 'timestamp' => $timestamp, - 'offsets' => $offsets['data'], - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Produce.php b/Kafka/Protocol/Produce.php deleted file mode 100644 index 55ca6579..00000000 --- a/Kafka/Protocol/Produce.php +++ /dev/null @@ -1,276 +0,0 @@ -requestHeader('kafka-php', 0, self::PRODUCE_REQUEST); - $data = self::pack(self::BIT_B16, $payloads['required_ack']); - $data .= self::pack(self::BIT_B32, $payloads['timeout']); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeProcudeTopic'), self::COMPRESSION_NONE); - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode produce response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $version = $this->getApiVersion(self::PRODUCE_REQUEST); - $ret = $this->decodeArray(substr($data, $offset), array($this, 'produceTopicPair'), $version); - $offset += $ret['length']; - $throttleTime = 0; - if ($version == self::API_VERSION2) { - $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - } - return array('throttleTime' => $throttleTime, 'data' => $ret['data']); - } - - // }}} - // {{{ protected function encodeMessageSet() - - /** - * encode message set - * N.B., MessageSets are not preceded by an int32 like other array elements - * in the protocol. - * - * @param array $messages - * @param int $compression - * @return string - * @static - * @access public - */ - protected function encodeMessageSet($messages, $compression = self::COMPRESSION_NONE) - { - if (!is_array($messages)) { - $messages = array($messages); - } - - $data = ''; - $next = 0; - foreach ($messages as $message) { - $tmpMessage = $this->encodeMessage($message, $compression); - - // int64 -- message offset Message - //This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. (see more details about compressed messages in Kafka below) - $data .= self::pack(self::BIT_B64, $next) . self::encodeString($tmpMessage, self::PACK_INT32); - $next++; - } - return $data; - } - - // }}} - // {{{ protected function encodeMessage() - - /** - * encode signal message - * - * @param string $message - * @param int $compression - * @return string - * @static - * @access protected - */ - protected function encodeMessage($message, $compression = self::COMPRESSION_NONE) - { - // int8 -- magic int8 -- attribute - $version = $this->getApiVersion(self::PRODUCE_REQUEST); - $magic = ($version == self::API_VERSION2) ? self::MESSAGE_MAGIC_VERSION0 : self::MESSAGE_MAGIC_VERSION1; - $data = self::pack(self::BIT_B8, $magic); - $data .= self::pack(self::BIT_B8, $compression); - - $key = ''; - if (is_array($message)) { - $key = $message['key']; - $message = $message['value']; - } - // message key - $data .= self::encodeString($key, self::PACK_INT32); - - // message value - $data .= self::encodeString($message, self::PACK_INT32, $compression); - - $crc = crc32($data); - - // int32 -- crc code string data - $message = self::pack(self::BIT_B32, $crc) . $data; - - return $message; - } - - // }}} - // {{{ protected function encodeProcudePartion() - - /** - * encode signal part - * - * @param $values - * @param $compression - * @return string - * @internal param $partions - * @access protected - */ - protected function encodeProcudePartion($values, $compression) - { - if (!isset($values['partition_id'])) { - throw new \Kafka\Exception\Protocol('given produce data invalid. `partition_id` is undefined.'); - } - - if (!isset($values['messages']) || empty($values['messages'])) { - throw new \Kafka\Exception\Protocol('given produce data invalid. `messages` is undefined.'); - } - - $data = self::pack(self::BIT_B32, $values['partition_id']); - $data .= self::encodeString($this->encodeMessageSet($values['messages'], $compression), self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ protected function encodeProcudeTopic() - - /** - * encode signal topic - * - * @param $values - * @param $compression - * @return string - * @internal param $partions - * @access protected - */ - protected function encodeProcudeTopic($values, $compression) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given produce data invalid. `topic_name` is undefined.'); - } - - if (!isset($values['partitions']) || empty($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given produce data invalid. `partitions` is undefined.'); - } - - $topic = self::encodeString($values['topic_name'], self::PACK_INT16); - $partitions = self::encodeArray($values['partitions'], array($this, 'encodeProcudePartion'), $compression); - - return $topic . $partitions; - } - - // }}} - // {{{ protected function produceTopicPair() - - /** - * decode produce topic pair response - * - * @access protected - * @return array - */ - protected function produceTopicPair($data, $version) - { - $offset = 0; - $topicInfo = $this->decodeString($data, self::BIT_B16); - $offset += $topicInfo['length']; - $ret = $this->decodeArray(substr($data, $offset), array($this, 'producePartitionPair'), $version); - $offset += $ret['length']; - - return array('length' => $offset, 'data' => array( - 'topicName' => $topicInfo['data'], - 'partitions'=> $ret['data'], - )); - } - - // }}} - // {{{ protected function producePartitionPair() - - /** - * decode produce partition pair response - * - * @access protected - * @return array - */ - protected function producePartitionPair($data, $version) - { - $offset = 0; - $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - $partitionOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - $timestamp = 0; - if ($version == self::API_VERSION2) { - $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - } - - return array( - 'length' => $offset, - 'data' => array( - 'partition' => $partitionId, - 'errorCode' => $errorCode, - 'offset' => $offset, - 'timestamp' => $timestamp, - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/Protocol.php b/Kafka/Protocol/Protocol.php deleted file mode 100644 index cc6d74b1..00000000 --- a/Kafka/Protocol/Protocol.php +++ /dev/null @@ -1,679 +0,0 @@ -version = $version; - } - - // }}} - // {{{ public static function Khex2bin() - - /** - * hex to bin - * - * @param string $string - * @static - * @access protected - * @return string (raw) - */ - public static function Khex2bin($string) - { - if (function_exists('\hex2bin')) { - return \hex2bin($string); - } else { - $bin = ''; - $len = strlen($string); - for ($i = 0; $i < $len; $i += 2) { - $bin .= pack('H*', substr($string, $i, 2)); - } - - return $bin; - } - } - - // }}} - // {{{ public static function unpack() - - /** - * Unpack a bit integer as big endian long - * - * @static - * @access public - * @param $type - * @param $bytes - * @return int - */ - public static function unpack($type, $bytes) - { - $result = array(); - self::checkLen($type, $bytes); - if ($type == self::BIT_B64) { - $set = unpack($type, $bytes); - $result = ($set[1] & 0xFFFFFFFF) << 32 | ($set[2] & 0xFFFFFFFF); - } elseif ($type == self::BIT_B16_SIGNED) { - // According to PHP docs: 's' = signed short (always 16 bit, machine byte order) - // So lets unpack it.. - $set = unpack($type, $bytes); - - // But if our system is little endian - if (self::isSystemLittleEndian()) { - // We need to flip the endianess because coming from kafka it is big endian - $set = self::convertSignedShortFromLittleEndianToBigEndian($set); - } - $result = $set; - } else { - $result = unpack($type, $bytes); - } - - return is_array($result) ? array_shift($result) : $result; - } - - // }}} - // {{{ public static function pack() - - /** - * pack a bit integer as big endian long - * - * @static - * @access public - * @param $type - * @param $data - * @return int - */ - public static function pack($type, $data) - { - if ($type == self::BIT_B64) { - if ($data == -1) { // -1L - $data = self::Khex2bin('ffffffffffffffff'); - } elseif ($data == -2) { // -2L - $data = self::Khex2bin('fffffffffffffffe'); - } else { - $left = 0xffffffff00000000; - $right = 0x00000000ffffffff; - - $l = ($data & $left) >> 32; - $r = $data & $right; - $data = pack($type, $l, $r); - } - } else { - $data = pack($type, $data); - } - - return $data; - } - - // }}} - // {{{ protected static function checkLen() - - /** - * check unpack bit is valid - * - * @param string $type - * @param string(raw) $bytes - * @static - * @access protected - * @return void - */ - protected static function checkLen($type, $bytes) - { - $len = 0; - switch ($type) { - case self::BIT_B64: - $len = 8; - break; - case self::BIT_B32: - $len = 4; - break; - case self::BIT_B16: - $len = 2; - break; - case self::BIT_B16_SIGNED: - $len = 2; - break; - case self::BIT_B8: - $len = 1; - break; - } - - if (strlen($bytes) != $len) { - throw new \Kafka\Exception\Protocol('unpack failed. string(raw) length is ' . strlen($bytes) . ' , TO ' . $type); - } - } - - // }}} - // {{{ public static function isSystemLittleEndian() - - /** - * Determines if the computer currently running this code is big endian or little endian. - * - * @access public - * @return bool - false if big endian, true if little endian - */ - public static function isSystemLittleEndian() - { - // If we don't know if our system is big endian or not yet... - if (is_null(self::$isLittleEndianSystem)) { - // Lets find out - list($endiantest) = array_values(unpack('L1L', pack('V', 1))); - if ($endiantest != 1) { - // This is a big endian system - self::$isLittleEndianSystem = false; - } else { - // This is a little endian system - self::$isLittleEndianSystem = true; - } - } - - return self::$isLittleEndianSystem; - } - - // }}} - // {{{ public static function convertSignedShortFromLittleEndianToBigEndian() - - /** - * Converts a signed short (16 bits) from little endian to big endian. - * - * @param int[] $bits - * @access public - * @return array - */ - public static function convertSignedShortFromLittleEndianToBigEndian($bits) - { - foreach ($bits as $index => $bit) { - - // get LSB - $lsb = $bit & 0xff; - - // get MSB - $msb = $bit >> 8 & 0xff; - - // swap bytes - $bit = $lsb <<8 | $msb; - - if ($bit >= 32768) { - $bit -= 65536; - } - $bits[$index] = $bit; - } - return $bits; - } - - // }}} - // {{{ public function getApiVersion() - - /** - * Get kafka api version according to specifiy kafka broker version - * - * @param int kafka api key - * @access public - * @return int - */ - public function getApiVersion($apikey) - { - switch ($apikey) { - case self::METADATA_REQUEST: - return self::API_VERSION0; - case self::PRODUCE_REQUEST: - if (version_compare($this->version, '0.10.0') >= 0) { - return self::API_VERSION2; - } elseif (version_compare($this->version, '0.9.0') >= 0) { - return self::API_VERSION1; - } else { - return self::API_VERSION0; - } - case self::FETCH_REQUEST: - if (version_compare($this->version, '0.10.0') >= 0) { - return self::API_VERSION2; - } elseif (version_compare($this->version, '0.9.0') >= 0) { - return self::API_VERSION1; - } else { - return self::API_VERSION0; - } - case self::OFFSET_REQUEST: - // todo - return self::API_VERSION0; - if (version_compare($this->version, '0.10.1.0') >= 0) { - return self::API_VERSION1; - } else { - return self::API_VERSION0; - } - case self::GROUP_COORDINATOR_REQUEST: - return self::API_VERSION0; - case self::OFFSET_COMMIT_REQUEST: - if (version_compare($this->version, '0.9.0') >= 0) { - return self::API_VERSION2; - } elseif (version_compare($this->version, '0.8.2') >= 0) { - return self::API_VERSION1; - } else { - return self::API_VERSION0; // supported in 0.8.1 or later - } - case self::OFFSET_FETCH_REQUEST: - if (version_compare($this->version, '0.8.2') >= 0) { - return self::API_VERSION1; // Offset Fetch Request v1 will fetch offset from Kafka - } else { - return self::API_VERSION0;//Offset Fetch Request v0 will fetch offset from zookeeper - } - case self::JOIN_GROUP_REQUEST: - if (version_compare($this->version, '0.10.1.0') >= 0) { - return self::API_VERSION1; - } else { - return self::API_VERSION0; // supported in 0.9.0.0 and greater - } - case self::SYNC_GROUP_REQUEST: - return self::API_VERSION0; - case self::HEART_BEAT_REQUEST: - return self::API_VERSION0; - case self::LEAVE_GROUP_REQUEST: - return self::API_VERSION0; - case self::LIST_GROUPS_REQUEST: - return self::API_VERSION0; - case self::DESCRIBE_GROUPS_REQUEST: - return self::API_VERSION0; - } - - // default - return self::API_VERSION0; - } - - // }}} - // {{{ public static function getApiText() - - /** - * Get kafka api text - * - * @param int kafka api key - * @access public - * @return string - */ - public static function getApiText($apikey) - { - $apis = array( - self::PRODUCE_REQUEST => 'ProduceRequest', - self::FETCH_REQUEST => 'FetchRequest', - self::OFFSET_REQUEST => 'OffsetRequest', - self::METADATA_REQUEST => 'MetadataRequest', - self::OFFSET_COMMIT_REQUEST => 'OffsetCommitRequest', - self::OFFSET_FETCH_REQUEST => 'OffsetFetchRequest', - self::GROUP_COORDINATOR_REQUEST => 'GroupCoordinatorRequest', - self::JOIN_GROUP_REQUEST => 'JoinGroupRequest', - self::HEART_BEAT_REQUEST => 'HeartbeatRequest', - self::LEAVE_GROUP_REQUEST => 'LeaveGroupRequest', - self::SYNC_GROUP_REQUEST => 'SyncGroupRequest', - self::DESCRIBE_GROUPS_REQUEST => 'DescribeGroupsRequest', - self::LIST_GROUPS_REQUEST => 'ListGroupsRequest', - ); - return $apis[$apikey]; - } - - // }}} - // {{{ public function requestHeader() - - /** - * get request header - * - * @param string $clientId - * @param integer $correlationId - * @param integer $apiKey - * @access public - * @return string - */ - public function requestHeader($clientId, $correlationId, $apiKey) - { - // int16 -- apiKey int16 -- apiVersion int32 correlationId - $binData = self::pack(self::BIT_B16, $apiKey); - $binData .= self::pack(self::BIT_B16, $this->getApiVersion($apiKey)); - $binData .= self::pack(self::BIT_B32, $correlationId); - - // concat client id - $binData .= self::encodeString($clientId, self::PACK_INT16); - $msg = sprintf('ClientId: %s ApiKey: %s ApiVersion: %s', $clientId, self::getApiText($apiKey), $this->getApiVersion($apiKey)); - $this->debug('Start Request ' . $msg); - - return $binData; - } - - // }}} - // {{{ public static function encodeString() - - /** - * encode pack string type - * - * @param string $string - * @param int $bytes self::PACK_INT32: int32 big endian order. self::PACK_INT16: int16 big endian order. - * @param int $compression - * @return string - * @static - * @access public - */ - public static function encodeString($string, $bytes, $compression = self::COMPRESSION_NONE) - { - $packLen = ($bytes == self::PACK_INT32) ? self::BIT_B32 : self::BIT_B16; - switch ($compression) { - case self::COMPRESSION_NONE: - break; - case self::COMPRESSION_GZIP: - $string = \gzencode($string); - break; - case self::COMPRESSION_SNAPPY: - // todo - throw new \Kafka\Exception\NotSupported('SNAPPY compression not yet implemented'); - default: - throw new \Kafka\Exception\NotSupported('Unknown compression flag: ' . $compression); - } - return self::pack($packLen, strlen($string)) . $string; - } - - // }}} - // {{{ public static function encodeArray() - - /** - * encode key array - * - * @param array $array - * @param Callable $func - * @param null $options - * @return string - * @static - * @access public - */ - public static function encodeArray(array $array, $func, $options = null) - { - if (!is_callable($func, false)) { - throw new \Kafka\Exception\Protocol('Encode array failed, given function is not callable.'); - } - - $arrayCount = count($array); - - $body = ''; - foreach ($array as $value) { - if (!is_null($options)) { - $body .= call_user_func($func, $value, $options); - } else { - $body .= call_user_func($func, $value); - } - } - - return self::pack(self::BIT_B32, $arrayCount) . $body; - } - - // }}} - // {{{ public function decodeString() - - /** - * decode unpack string type - * - * @param bytes $data - * @param int $bytes self::BIT_B32: int32 big endian order. self::BIT_B16: int16 big endian order. - * @param int $compression - * @return string - * @access public - */ - public function decodeString($data, $bytes, $compression = self::COMPRESSION_NONE) - { - $offset = ($bytes == self::BIT_B32) ? 4 : 2; - $packLen = self::unpack($bytes, substr($data, 0, $offset)); // int16 topic name length - if ($packLen == 4294967295) { // uint32(4294967295) is int32 (-1) - $packLen = 0; - } - - if ($packLen == 0) { - return array('length' => $offset, 'data' => ''); - } - - $data = substr($data, $offset, $packLen); - $offset += $packLen; - - switch ($compression) { - case self::COMPRESSION_NONE: - break; - case self::COMPRESSION_GZIP: - $data = \gzdecode($data); - break; - case self::COMPRESSION_SNAPPY: - // todo - throw new \Kafka\Exception\NotSupported('SNAPPY compression not yet implemented'); - default: - throw new \Kafka\Exception\NotSupported('Unknown compression flag: ' . $compression); - } - return array('length' => $offset, 'data' => $data); - } - - // }}} - // {{{ public function decodeArray() - - /** - * decode key array - * - * @param array $array - * @param Callable $func - * @param null $options - * @return string - * @access public - */ - public function decodeArray($data, $func, $options = null) - { - $offset = 0; - $arrayCount = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - - if (!is_callable($func, false)) { - throw new \Kafka\Exception\Protocol('Decode array failed, given function is not callable.'); - } - - $result = array(); - for ($i = 0; $i < $arrayCount; $i++) { - $value = substr($data, $offset); - if (!is_null($options)) { - $ret = call_user_func($func, $value, $options); - } else { - $ret = call_user_func($func, $value); - } - - if (!is_array($ret) && $ret === false) { - break; - } - - if (!isset($ret['length']) || !isset($ret['data'])) { - throw new \Kafka\Exception\Protocol('Decode array failed, given function return format is invliad'); - } - if ($ret['length'] == 0) { - continue; - } - - $offset += $ret['length']; - $result[] = $ret['data']; - } - - return array('length' => $offset, 'data' => $result); - } - - // }}} - // {{{ public function decodePrimitiveArray() - - /** - * decode primitive type array - * - * @param bytes[] $data - * @param bites $bites - * @return array - * @access public - */ - public function decodePrimitiveArray($data, $bites) - { - $offset = 0; - $arrayCount = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - if ($arrayCount == 4294967295) { - $arrayCount = 0; - } - - $result = array(); - - for ($i = 0; $i < $arrayCount; $i++) { - if ($bites == self::BIT_B64) { - $result[] = self::unpack(self::BIT_B64, substr($data, $offset, 8)); - $offset += 8; - } elseif ($bites == self::BIT_B32) { - $result[] = self::unpack(self::BIT_B32, substr($data, $offset, 4)); - $offset += 4; - } elseif (in_array($bites, array(self::BIT_B16, self::BIT_B16_SIGNED))) { - $result[] = self::unpack($bites, substr($data, $offset, 2)); - $offset += 2; - } elseif ($bites == self::BIT_B8) { - $result[] = self::unpack($bites, substr($data, $offset, 1)); - $offset += 1; - } - } - - return array('length' => $offset, 'data' => $result); - } - - // }}} - // }}} -} diff --git a/Kafka/Protocol/SyncGroup.php b/Kafka/Protocol/SyncGroup.php deleted file mode 100644 index 1993aeff..00000000 --- a/Kafka/Protocol/SyncGroup.php +++ /dev/null @@ -1,212 +0,0 @@ -requestHeader('kafka-php', self::SYNC_GROUP_REQUEST, self::SYNC_GROUP_REQUEST); - $data = self::encodeString($payloads['group_id'], self::PACK_INT16); - $data .= self::pack(self::BIT_B32, $payloads['generation_id']); - $data .= self::encodeString($payloads['member_id'], self::PACK_INT16); - $data .= self::encodeArray($payloads['data'], array($this, 'encodeGroupAssignment')); - - $data = self::encodeString($header . $data, self::PACK_INT32); - - return $data; - } - - // }}} - // {{{ public function decode() - - /** - * decode group response - * - * @access public - * @return array - */ - public function decode($data) - { - $offset = 0; - $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2)); - $offset += 2; - - $memberAssignments = $this->decodeString(substr($data, $offset), self::BIT_B32); - $offset += $memberAssignments['length']; - - $memberAssignment = $memberAssignments['data']; - if (strlen($memberAssignment)) { - $memberAssignmentOffset = 0; - $version = self::unpack(self::BIT_B16_SIGNED, substr($memberAssignment, $memberAssignmentOffset, 2)); - $memberAssignmentOffset += 2; - $partitionAssignments = $this->decodeArray(substr($memberAssignment, $memberAssignmentOffset), - array($this, 'syncGroupResponsePartition')); - $memberAssignmentOffset += $partitionAssignments['length']; - $userData = $this->decodeString(substr($memberAssignment, $memberAssignmentOffset), self::BIT_B32); - } else { - return array( - 'errorCode' => $errorCode, - ); - } - - return array( - 'errorCode' => $errorCode, - 'partitionAssignments' => $partitionAssignments['data'], - 'version' => $version, - 'userData' => $userData['data'], - ); - } - - // }}} - // {{{ protected function encodeGroupAssignment() - - /** - * encode group assignment protocol - * - * @param partions - * @access protected - * @return string - */ - protected function encodeGroupAssignment($values) - { - if (!isset($values['version'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `version` is undefined.'); - } - if (!isset($values['member_id'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `member_id` is undefined.'); - } - - if (!isset($values['assignments'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `assignments` is undefined.'); - } - if (!isset($values['user_data'])) { - $values['user_data'] = ''; - } - - $memberId = self::encodeString($values['member_id'], self::PACK_INT16); - - $data = self::pack(self::BIT_B16, 0); - $data .= self::encodeArray($values['assignments'], array($this, 'encodeGroupAssignmentTopic')); - $data .= self::encodeString($values['user_data'], self::PACK_INT32); - - return $memberId . self::encodeString($data, self::PACK_INT32); - } - - // }}} - // {{{ protected function encodeGroupAssignmentTopic() - - /** - * encode group assignment topic protocol - * - * @param partions - * @access protected - * @return string - */ - protected function encodeGroupAssignmentTopic($values) - { - if (!isset($values['topic_name'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `topic_name` is undefined.'); - } - if (!isset($values['partitions'])) { - throw new \Kafka\Exception\Protocol('given data invalid. `partitions` is undefined.'); - } - - $topicName = self::encodeString($values['topic_name'], self::PACK_INT16); - - $partitions = self::encodeArray($values['partitions'], array($this, 'encodeGroupAssignmentTopicPartition')); - - return $topicName . $partitions; - } - - // }}} - // {{{ protected function encodeGroupAssignmentTopicPartition() - - /** - * encode group assignment topic protocol - * - * @param partions - * @access protected - * @return string - */ - protected function encodeGroupAssignmentTopicPartition($values) - { - return self::pack(self::BIT_B32, $values); - } - - // }}} - // {{{ protected function syncGroupResponsePartition() - - /** - * decode sync group partition response - * - * @access protected - * @return array - */ - protected function syncGroupResponsePartition($data) - { - $offset = 0; - $topicName = $this->decodeString(substr($data, $offset), self::BIT_B16); - $offset += $topicName['length']; - $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32); - $offset += $partitions['length']; - - return array( - 'length' => $offset, - 'data' => array( - 'topicName' => $topicName['data'], - 'partitions' => $partitions['data'], - ) - ); - } - - // }}} - // }}} -} diff --git a/Kafka/SingletonTrait.php b/Kafka/SingletonTrait.php deleted file mode 100644 index b2dd5ced..00000000 --- a/Kafka/SingletonTrait.php +++ /dev/null @@ -1,76 +0,0 @@ -host = $host; - $this->port = $port; - $this->setRecvTimeoutSec($recvTimeoutSec); - $this->setRecvTimeoutUsec($recvTimeoutUsec); - $this->setSendTimeoutSec($sendTimeoutSec); - $this->setSendTimeoutUsec($sendTimeoutUsec); - } - - /** - * @param float $sendTimeoutSec - */ - public function setSendTimeoutSec(float $sendTimeoutSec) - { - $this->sendTimeoutSec = $sendTimeoutSec; - } - - /** - * @param float $sendTimeoutUsec - */ - public function setSendTimeoutUsec(float $sendTimeoutUsec) - { - $this->sendTimeoutUsec = $sendTimeoutUsec; - } - - /** - * @param float $recvTimeoutSec - */ - public function setRecvTimeoutSec(float $recvTimeoutSec) - { - $this->recvTimeoutSec = $recvTimeoutSec; - } - - /** - * @param float $recvTimeoutUsec - */ - public function setRecvTimeoutUsec(float $recvTimeoutUsec) - { - $this->recvTimeoutUsec = $recvTimeoutUsec; - } - - /** - * @param int $number - */ - public function setMaxWriteAttempts(int $number) - { - $this->maxWriteAttempts = $number; - } - - - /** - * Connects the socket - * - * @access public - * @return void - */ - public function connect() - { - if (!$this->isSocketDead()) { - return; - } - - if (empty($this->host)) { - throw new \Kafka\Exception('Cannot open null host.'); - } - if ($this->port <= 0) { - throw new \Kafka\Exception('Cannot open without port.'); - } - - $this->stream = @fsockopen( - $this->host, - $this->port, - $errno, - $errstr, - $this->sendTimeoutSec + ($this->sendTimeoutUsec / 1000000) - ); - - if ($this->stream == false) { - $error = 'Could not connect to ' - . $this->host . ':' . $this->port - . ' (' . $errstr . ' [' . $errno . '])'; - throw new \Kafka\Exception($error); - } - - stream_set_blocking($this->stream, 0); - stream_set_read_buffer($this->stream, 0); - - $this->readWatcher = \Amp\onReadable($this->stream, function () { - do { - if (!$this->isSocketDead()) { - $newData = @fread($this->stream, self::READ_MAX_LEN); - } else { - $this->reconnect(); - return; - } - if ($newData) { - $this->read($newData); - $newData = true; - } - } while ($newData); - }); - - $this->writeWatcher = \Amp\onWritable($this->stream, function () { - $this->write(); - }, array('enable' => false)); - } - - - /** - * reconnect the socket - * - * @access public - * @return void - */ - public function reconnect() - { - $this->close(); - $this->connect(); - } - - - /** - * get the socket - * - * @access public - * @return void - */ - public function getSocket() - { - return $this->stream; - } - - - public $onReadable; - - /** - * set on readable callback function - * - * @access public - * @param \Closure $read - * @return void - */ - public function SetOnReadable(\Closure $read) - { - $this->onReadable = $read; - } - - - /** - * close the socket - * - * @access public - * @return void - */ - public function close() - { - \Amp\cancel($this->readWatcher); - \Amp\cancel($this->writeWatcher); - if (is_resource($this->stream)) { - fclose($this->stream); - } - $this->readBuffer = ''; - $this->writeBuffer = ''; - $this->readNeedLength = 0; - } - - /** - * checks if the socket is a valid resource - * - * @access public - * @return boolean - */ - public function isResource() - { - return is_resource($this->stream); - } - - - /** - * Read from the socket at most $len bytes. - * - * This method will not wait for all the requested data, it will return as - * soon as any data is received. - * - * @param $data - */ - public function read($data) - { - $this->readBuffer .= $data; - do { - if ($this->readNeedLength == 0) { - if (strlen($this->readBuffer) < 4) { - return; - } - $dataLen = \Kafka\Protocol\Protocol::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($this->readBuffer, 0, 4)); - $this->readNeedLength = $dataLen; - $this->readBuffer = substr($this->readBuffer, 4); - } - - if (strlen($this->readBuffer) < $this->readNeedLength) { - return; - } - $data = substr($this->readBuffer, 0, $this->readNeedLength); - - $this->readBuffer = substr($this->readBuffer, $this->readNeedLength); - $this->readNeedLength = 0; - call_user_func($this->onReadable, $data, (int)$this->stream); - } while (strlen($this->readBuffer)); - } - - - /** - * Write to the socket. - * - * @param null $data - */ - public function write($data = null) - { - if ($data != null) { - $this->writeBuffer .= $data; - } - $bytesToWrite = strlen($this->writeBuffer); - $bytesWritten = @fwrite($this->stream, $this->writeBuffer); - - if ($bytesToWrite === $bytesWritten) { - \Amp\disable($this->writeWatcher); - } elseif ($bytesWritten >= 0) { - \Amp\enable($this->writeWatcher); - } elseif ($this->isSocketDead()) { - $this->reconnect(); - } - $this->writeBuffer = substr($this->writeBuffer, $bytesWritten); - } - - - /** - * check the stream is close - * - * @return bool - */ - protected function isSocketDead() - { - return !is_resource($this->stream) || @feof($this->stream); - } -} diff --git a/Kafka/SocketSync.php b/Kafka/SocketSync.php deleted file mode 100644 index bbe07169..00000000 --- a/Kafka/SocketSync.php +++ /dev/null @@ -1,337 +0,0 @@ -host = $host; - $this->port = $port; - $this->setRecvTimeoutSec($recvTimeoutSec); - $this->setRecvTimeoutUsec($recvTimeoutUsec); - $this->setSendTimeoutSec($sendTimeoutSec); - $this->setSendTimeoutUsec($sendTimeoutUsec); - } - - /** - * @param float $sendTimeoutSec - */ - public function setSendTimeoutSec(float $sendTimeoutSec) - { - $this->sendTimeoutSec = $sendTimeoutSec; - } - - /** - * @param float $sendTimeoutUsec - */ - public function setSendTimeoutUsec(float $sendTimeoutUsec) - { - $this->sendTimeoutUsec = $sendTimeoutUsec; - } - - /** - * @param float $recvTimeoutSec - */ - public function setRecvTimeoutSec(float $recvTimeoutSec) - { - $this->recvTimeoutSec = $recvTimeoutSec; - } - - /** - * @param float $recvTimeoutUsec - */ - public function setRecvTimeoutUsec(float $recvTimeoutUsec) - { - $this->recvTimeoutUsec = $recvTimeoutUsec; - } - - /** - * @param int $number - */ - public function setMaxWriteAttempts(int $number) - { - $this->maxWriteAttempts = $number; - } - - - /** - * Optional method to set the internal stream handle - * - * @static - * @access public - * @param $stream - * @return SocketSync - */ - public static function createFromStream($stream) - { - $socket = new self('localhost', 0); - $socket->setStream($stream); - return $socket; - } - - - /** - * Optional method to set the internal stream handle - * - * @param mixed $stream - * @access public - * @return void - */ - public function setStream($stream) - { - $this->stream = $stream; - } - - - /** - * Connects the socket - * - * @access public - * @return Coroutine\Socket - */ - public function connect() - { - if (Context::hasContext(self::SOCKET_NAME)) { - return Context::getContext(self::SOCKET_NAME); - } - if (empty($this->host)) { - throw new Exception('Cannot open null host.'); - } - if ($this->port <= 0) { - throw new Exception('Cannot open without port.'); - } - $stream = new \Swoole\Coroutine\Socket(AF_INET, SOCK_STREAM); - if (!$stream->connect($this->host, $this->port)) { - $error = 'Could not connect to ' - . $this->host . ':' . $this->port - . ' (' . $stream->errMsg . ' [' . $stream->errMsg . '])'; - throw new Exception($error); - } - return Context::setContext(self::SOCKET_NAME, $stream); - } - - - /** - * close the socket - * - * @access public - * @return void - */ - public function close() - { - - } - - /** - * checks if the socket is a valid resource - * - * @access public - * @return boolean - */ - public function isResource() - { - if (!Context::hasContext(self::SOCKET_NAME)) { - return false; - } - return Context::getContext(self::SOCKET_NAME)->checkLiveness(); - } - - /** - * Read from the socket at most $len bytes. - * - * This method will not wait for all the requested data, it will return as - * soon as any data is received. - * - * @param integer $len Maximum number of bytes to read. - * @param boolean $verifyExactLength Throw an exception if the number of read bytes is less than $len - * - * @return string Binary data - * @throws Exception - */ - public function read(int $len, $verifyExactLength = false) - { - if ($len > self::READ_MAX_LEN) { - throw new Exception('Could not read ' . $len . ' bytes from stream, length too longer.'); - } - $stream = $this->connect(); - - $null = null; - $remainingBytes = $len; - $data = $chunk = ''; - while ($remainingBytes > 0) { - $chunk = $stream->recv($remainingBytes); - if ($chunk === false) { - throw new Exception('Could not read ' . $len . ' bytes from stream (no data)'); - } - if (strlen($chunk) === 0) { - continue; - } - $data .= $chunk; - $remainingBytes -= strlen($chunk); - } - if ($len === $remainingBytes || ($verifyExactLength && $len !== strlen($data))) { - throw new Exception('Read ' . strlen($data) . ' bytes instead of the requested ' . $len . ' bytes'); - } - return $data; - } - - /** - * Write to the socket. - * - * @param string $buf The data to write - * - * @return integer - * @throws Exception - * @throws \Exception - */ - public function write(string $buf) - { - $null = null; - - $failedWriteAttempts = 0; - $written = 0; - $buflen = strlen($buf); - - $stream = $this->connect(); - while ($written < $buflen) { - if ($buflen - $written > self::MAX_WRITE_BUFFER) { - $wrote = $stream->send(substr($buf, $written, self::MAX_WRITE_BUFFER), 1); - } else { - $wrote = $stream->send(substr($buf, $written), 1); - } - if ($wrote === -1 || $wrote === false) { - throw new \Kafka\Exception\Socket('Could not write ' . strlen($buf) . ' bytes to stream, completed writing only ' . $written . ' bytes'); - } elseif ($wrote === 0) { - $failedWriteAttempts++; - if ($failedWriteAttempts > $this->maxWriteAttempts) { - throw new \Kafka\Exception\Socket('After ' . $failedWriteAttempts . ' attempts could not write ' . strlen($buf) . ' bytes to stream, completed writing only ' . $written . ' bytes'); - } - } else { - $failedWriteAttempts = 0; - } - $written += $wrote; - } - return $written; - } - -}