as2252258/kiri-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

modify

Browse Source
This commit is contained in:
as2252258@163.com
2021-04-11 17:18:54 +08:00
parent 364c638d81
commit 6fbb0dea76
4 changed files with 245 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Annotation/Kafka.php
+11 -1
View File
@@ -4,6 +4,9 @@
namespace Annotation; namespace Annotation;
use Kafka\ConsumerInterface;
use Kafka\TaskContainer;
/** /**
* Class Kafka * Class Kafka
* @package Annotation * @package Annotation
@@ -16,7 +19,7 @@ namespace Annotation;
* Kafka constructor. * Kafka constructor.
* @param string $topic * @param string $topic
*/ */
public function __construct(public string $topic, public string $groupId, public ?string $brokers = null) public function __construct(public string $topic)
{ {
} }
@@ -28,6 +31,13 @@ namespace Annotation;
*/ */
public function execute(array $handler): mixed public function execute(array $handler): mixed
{ {
if (!($handler[0] instanceof ConsumerInterface)) {
return false;
}
$container = TaskContainer::getInstance();
$container->addConsumer($this->topic, [$handler[0], 'onHandler']);
return parent::execute($handler); // TODO: Change the autogenerated stub return parent::execute($handler); // TODO: Change the autogenerated stub
} }
HttpServer/Events/OnPipeMessage.php
+44 -2
View File
@@ -7,6 +7,9 @@ namespace HttpServer\Events;
use Annotation\Loader; use Annotation\Loader;
use Exception; use Exception;
use HttpServer\Abstracts\Callback; use HttpServer\Abstracts\Callback;
use Kafka\ConsumerInterface;
use Kafka\Struct;
use Kafka\TaskContainer;
use Snowflake\Event; use Snowflake\Event;
use Snowflake\Exception\ComponentException; use Snowflake\Exception\ComponentException;
use Snowflake\Snowflake; use Snowflake\Snowflake;
@@ -28,8 +31,47 @@ class OnPipeMessage extends Callback
*/ */
public function onHandler(Server $server, int $src_worker_id, $message) public function onHandler(Server $server, int $src_worker_id, $message)
{ {
$events = Snowflake::app()->getEvent(); try {
$events->trigger(Event::PIPE_MESSAGE, [$server, $src_worker_id, $message]); $swoole_unserialize = swoole_unserialize($message);
match ($swoole_unserialize['action'] ?? null) {
'kafka' => $this->onKafkaWorker($swoole_unserialize),
default => $this->onMessageWorker($server, $src_worker_id, $message)
};
} catch (\Throwable $exception) {
$this->addError($exception);
} finally {
fire(Event::SYSTEM_RESOURCE_RELEASES);
}
} }
/**
* @param $server
* @param $src_worker_id
* @param $message
* @throws \Exception
*/
private function onMessageWorker($server, $src_worker_id, $message)
{
fire(Event::PIPE_MESSAGE, [$server, $src_worker_id, $message]);
return 'success';
}
/**
* @param array $message
* @throws \ReflectionException
* @throws \Snowflake\Exception\NotFindClassException
*/
private function onKafkaWorker(array $message)
{
[$topic, $message] = $message['body'];
$container = TaskContainer::getInstance();
$container->process($topic, new Struct($topic, $message));
return 'success';
}
} }
Kafka/Kafka.php
+128 -131
View File
@@ -10,6 +10,7 @@ use RdKafka\ConsumerTopic;
use RdKafka\Exception; use RdKafka\Exception;
use RdKafka\KafkaConsumer; use RdKafka\KafkaConsumer;
use RdKafka\TopicConf; use RdKafka\TopicConf;
use Snowflake\Core\Json;
use Snowflake\Snowflake; use Snowflake\Snowflake;
use Swoole\Coroutine\Channel; use Swoole\Coroutine\Channel;
use Swoole\Process; use Swoole\Process;
@@ -22,160 +23,156 @@ use Throwable;
class Kafka extends \Snowflake\Process\Process class Kafka extends \Snowflake\Process\Process
{ {
protected Channel $channel; protected Channel $channel;
private int $maxLength = 5000; private int $maxLength = 5000;
/** /**
* @param Process $process * @param Process $process
* @throws \Exception * @throws \Exception
*/ */
public function onHandler(Process $process): void public function onHandler(Process $process): void
{ {
$this->waite(swoole_unserialize($process->read())); $this->waite(swoole_unserialize($process->read()));
} }
/** /**
* @param array $kafkaServer * @param array $kafkaServer
* @throws \Exception * @throws \Exception
*/ */
private function waite(array $kafkaServer) private function waite(array $kafkaServer)
{ {
try { try {
name($this->pid, 'Kafka Consumer ' . $kafkaServer['topic']); name($this->pid, 'Kafka Consumer ' . $kafkaServer['topic']);
[$config, $topic, $conf] = $this->kafkaConfig($kafkaServer); [$config, $topic, $conf] = $this->kafkaConfig($kafkaServer);
if (empty($config) && empty($topic) && empty($conf)) { if (empty($config) && empty($topic) && empty($conf)) {
return; return;
} }
$objRdKafka = new Consumer($config); $objRdKafka = new Consumer($config);
$topic = $objRdKafka->newTopic($kafkaServer['topic'], $topic); $topic = $objRdKafka->newTopic($kafkaServer['topic'], $topic);
$topic->consumeStart(0, RD_KAFKA_OFFSET_STORED); $topic->consumeStart(0, RD_KAFKA_OFFSET_STORED);
do { do {
$this->resolve($topic, $conf['interval'] ?? 1000); $this->resolve($topic, $conf['interval'] ?? 1000);
} while (true); } while (true);
} catch (Throwable $exception) { } catch (Throwable $exception) {
logger()->addError($exception, 'throwable'); logger()->addError($exception, 'throwable');
} }
} }
/** /**
* @param ConsumerTopic $topic * @param ConsumerTopic $topic
* @param $interval * @param $interval
* @throws \Exception * @throws \Exception
*/ */
private function resolve(ConsumerTopic $topic, $interval) private function resolve(ConsumerTopic $topic, $interval)
{ {
try { try {
$message = $topic->consume(0, $interval); $message = $topic->consume(0, $interval);
if (empty($message)) { if (empty($message)) {
return; return;
} }
if ($message->err == RD_KAFKA_RESP_ERR_NO_ERROR) { if ($message->err == RD_KAFKA_RESP_ERR_NO_ERROR) {
$this->handlerExecute($message->topic_name, $message); $this->handlerExecute($message->topic_name, $message);
} else if ($message->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { } else if ($message->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
$this->application->warning('No more messages; will wait for more'); $this->application->warning('No more messages; will wait for more');
} else if ($message->err == RD_KAFKA_RESP_ERR__TIMED_OUT) { } else if ($message->err == RD_KAFKA_RESP_ERR__TIMED_OUT) {
$this->application->error('Kafka Timed out'); $this->application->error('Kafka Timed out');
} else { } else {
$this->application->error($message->errstr()); $this->application->error($message->errstr());
} }
} catch (Throwable $exception) { } catch (Throwable $exception) {
logger()->addError($exception, 'throwable'); logger()->addError($exception, 'throwable');
} }
} }
/** /**
* @param $topic * @param $topic
* @param $message * @param $message
* @throws \Exception * @throws \Exception
*/ */
protected function handlerExecute($topic, $message) protected function handlerExecute($topic, $message)
{ {
go(function () use ($topic, $message) { go(function () use ($topic, $message) {
try { try {
$topic = str_replace('-', '_', $topic); $server = Snowflake::app()->getSwoole();
$namespace = 'App\\Kafka\\' . ucfirst($topic) . 'Consumer'; $setting = $server->setting['worker_num'];
if (!class_exists($namespace)) {
return; $message = swoole_serialize(['action' => 'kafka', 'body' => [$topic, $message]]);
}
$class = Snowflake::createObject($namespace); $server->sendMessage($message, random_int(0, $setting));
if (!($class instanceof ConsumerInterface)) { } catch (Throwable $exception) {
return; logger()->addError($exception, 'throwable');
} }
$class->onHandler(Snowflake::createObject(Struct::class, [$topic, $message])); });
} catch (Throwable $exception) { }
logger()->addError($exception, 'throwable');
}
});
}
/** /**
* @param $kafka * @param $kafka
* @return array * @return array
* @throws \Exception * @throws \Exception
*/ */
private function kafkaConfig($kafka): array private function kafkaConfig($kafka): array
{ {
try { try {
$conf = new Conf(); $conf = new Conf();
$conf->setRebalanceCb([$this, 'rebalanced_cb']); $conf->setRebalanceCb([$this, 'rebalanced_cb']);
$conf->set('group.id', $kafka['groupId']); $conf->set('group.id', $kafka['groupId']);
$conf->set('metadata.broker.list', $kafka['brokers']); $conf->set('metadata.broker.list', $kafka['brokers']);
$conf->set('socket.timeout.ms', '30000'); $conf->set('socket.timeout.ms', '30000');
$this->application->debug('kafka listen groupId ' . $kafka['groupId']); $this->application->debug('kafka listen groupId ' . $kafka['groupId']);
$this->application->debug('kafka listen brokers ' . $kafka['brokers']); $this->application->debug('kafka listen brokers ' . $kafka['brokers']);
if (function_exists('pcntl_sigprocmask')) { if (function_exists('pcntl_sigprocmask')) {
pcntl_sigprocmask(SIG_BLOCK, array(SIGIO)); pcntl_sigprocmask(SIG_BLOCK, array(SIGIO));
$conf->set('internal.termination.signal', (string)SIGIO); $conf->set('internal.termination.signal', (string)SIGIO);
} }
$topicConf = new TopicConf(); $topicConf = new TopicConf();
$topicConf->set('auto.commit.enable', '1'); $topicConf->set('auto.commit.enable', '1');
$topicConf->set('auto.commit.interval.ms', '100'); $topicConf->set('auto.commit.interval.ms', '100');
//smallest：简单理解为从头开始消费， //smallest：简单理解为从头开始消费，
//largest：简单理解为从最新的开始消费 //largest：简单理解为从最新的开始消费
$topicConf->set('auto.offset.reset', 'smallest'); $topicConf->set('auto.offset.reset', 'smallest');
$topicConf->set('offset.store.path', 'kafka_offset.log'); $topicConf->set('offset.store.path', 'kafka_offset.log');
$topicConf->set('offset.store.method', 'broker'); $topicConf->set('offset.store.method', 'broker');
return [$conf, $topicConf, $kafka]; return [$conf, $topicConf, $kafka];
} catch (Throwable $exception) { } catch (Throwable $exception) {
logger()->addError($exception, 'throwable'); logger()->addError($exception, 'throwable');
return [null, null, null]; return [null, null, null];
} }
} }
/** /**
* @param KafkaConsumer $kafka * @param KafkaConsumer $kafka
* @param $err * @param $err
* @param array|null $partitions * @param array|null $partitions
* @throws Exception * @throws Exception
* @throws \Exception * @throws \Exception
*/ */
public function rebalanced_cb(KafkaConsumer $kafka, $err, array $partitions = null) public function rebalanced_cb(KafkaConsumer $kafka, $err, array $partitions = null)
{ {
if ($err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) { if ($err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
$kafka->assign($partitions); $kafka->assign($partitions);
} else if ($err == RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS) { } else if ($err == RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS) {
$kafka->assign(NULL); $kafka->assign(NULL);
} else { } else {
throw new \Exception($err); throw new \Exception($err);
} }
} }
} }
Kafka/TaskContainer.php
+62
View File
@@ -0,0 +1,62 @@
<?php
namespace Kafka;
use Snowflake\Abstracts\BaseObject;
/**
* Class TaskContainer
* @package Kafka
*/
class TaskContainer extends BaseObject
{
private array $_topics = [];
private static TaskContainer $container;
/**
* @return \Kafka\TaskContainer
*/
public static function getInstance(): TaskContainer
{
if (!(static::$container instanceof TaskContainer)) {
static::$container = new TaskContainer();
}
return static::$container;
}
/**
* @param $topic
* @param $handler
*/
public function addConsumer($topic, $handler)
{
if (isset($this->_topics[$topic])) {
return;
}
$this->_topics[$topic] = $handler;
}
/**
* @param $topic
* @param \Kafka\Struct $struct
*/
public function process($topic, Struct $struct)
{
$handler = $this->_topics[$topic] ?? null;
if (empty($handler)) {
return;
}
call_user_func($handler, $struct);
}
}