first-fix
This commit is contained in:
Alexander Abakumov committed
1 parent
6d87c1edc8
commit
565d9d673a
6284 files changed
+820267
-76
No files matched your search
@@ -0,0 +1,197 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
import config from 'config';
|
||||
import { _logger } from '../config/logger';
|
||||
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
|
||||
import { IQueue } from './queue.models';
|
||||
import {
|
||||
CreateQueueCommand,
|
||||
CreateQueueRequest,
|
||||
DeleteMessageBatchCommand,
|
||||
DeleteMessageBatchRequest,
|
||||
DeleteMessageBatchRequestEntry,
|
||||
ListQueuesCommand,
|
||||
ListQueuesResult,
|
||||
ReceiveMessageCommand,
|
||||
ReceiveMessageRequest,
|
||||
ReceiveMessageResult,
|
||||
SendMessageCommand,
|
||||
SendMessageRequest,
|
||||
SQSClient
|
||||
} from '@aws-sdk/client-sqs';
|
||||
import uuid from 'uuid-random';
|
||||
|
||||
export class AwsSqsTemplate implements IQueue {
|
||||
|
||||
private logger = _logger(`awsSqsTemplate`);
|
||||
private requestTopic: string = config.get('request_topic');
|
||||
private accessKeyId: string = config.get('aws_sqs.access_key_id');
|
||||
private secretAccessKey: string = config.get('aws_sqs.secret_access_key');
|
||||
private region: string = config.get('aws_sqs.region');
|
||||
private queueProperties: string = config.get('aws_sqs.queue_properties');
|
||||
private pollInterval = Number(config.get('js.response_poll_interval'));
|
||||
|
||||
private sqsClient: SQSClient;
|
||||
private requestQueueURL: string
|
||||
private queueUrls = new Map<string, string>();
|
||||
private queueAttributes: { [n: string]: string } = {
|
||||
FifoQueue: 'true'
|
||||
};
|
||||
private timer: NodeJS.Timer;
|
||||
|
||||
name = 'AWS SQS';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
async init() {
|
||||
this.sqsClient = new SQSClient({
|
||||
apiVersion: '2012-11-05',
|
||||
credentials: {
|
||||
accessKeyId: this.accessKeyId,
|
||||
secretAccessKey: this.secretAccessKey
|
||||
},
|
||||
region: this.region
|
||||
});
|
||||
|
||||
const queues = await this.getQueues();
|
||||
|
||||
if (queues.QueueUrls) {
|
||||
queues.QueueUrls.forEach(queueUrl => {
|
||||
const delimiterPosition = queueUrl.lastIndexOf('/');
|
||||
const queueName = queueUrl.substring(delimiterPosition + 1);
|
||||
this.queueUrls.set(queueName, queueUrl);
|
||||
});
|
||||
}
|
||||
|
||||
this.parseQueueProperties();
|
||||
|
||||
this.requestQueueURL = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(this.requestTopic)) || '';
|
||||
if (!this.requestQueueURL) {
|
||||
this.requestQueueURL = await this.createQueue(this.requestTopic);
|
||||
}
|
||||
|
||||
const messageProcessor = new JsInvokeMessageProcessor(this);
|
||||
|
||||
const params: ReceiveMessageRequest = {
|
||||
MaxNumberOfMessages: 10,
|
||||
QueueUrl: this.requestQueueURL,
|
||||
WaitTimeSeconds: Math.ceil(this.pollInterval / 10)
|
||||
};
|
||||
this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval);
|
||||
}
|
||||
|
||||
private async getAndProcessMessage(messageProcessor: JsInvokeMessageProcessor, params: ReceiveMessageRequest) {
|
||||
const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params));
|
||||
const messages = messagesResponse.Messages;
|
||||
|
||||
if (messages && messages.length > 0) {
|
||||
const entries: DeleteMessageBatchRequestEntry[] = [];
|
||||
|
||||
messages.forEach(message => {
|
||||
entries.push({
|
||||
Id: message.MessageId,
|
||||
ReceiptHandle: message.ReceiptHandle
|
||||
});
|
||||
messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || ''));
|
||||
});
|
||||
|
||||
const deleteBatch: DeleteMessageBatchRequest = {
|
||||
QueueUrl: this.requestQueueURL,
|
||||
Entries: entries
|
||||
};
|
||||
try {
|
||||
await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch))
|
||||
} catch (err: any) {
|
||||
this.logger.error("Failed to delete messages from queue.", err.message);
|
||||
}
|
||||
}
|
||||
this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval);
|
||||
}
|
||||
|
||||
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
|
||||
let msgBody = JSON.stringify(
|
||||
{
|
||||
key: msgKey,
|
||||
data: [...rawResponse],
|
||||
headers: headers
|
||||
});
|
||||
|
||||
let responseQueueUrl = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(responseTopic));
|
||||
|
||||
if (!responseQueueUrl) {
|
||||
responseQueueUrl = await this.createQueue(responseTopic);
|
||||
this.queueUrls.set(responseTopic, responseQueueUrl);
|
||||
}
|
||||
|
||||
let msgId = uuid();
|
||||
|
||||
let params: SendMessageRequest = {
|
||||
MessageBody: msgBody,
|
||||
QueueUrl: responseQueueUrl,
|
||||
MessageGroupId: msgId,
|
||||
MessageDeduplicationId: msgId
|
||||
};
|
||||
|
||||
return this.sqsClient.send(new SendMessageCommand(params))
|
||||
}
|
||||
|
||||
private async getQueues(): Promise<ListQueuesResult> {
|
||||
return this.sqsClient.send(new ListQueuesCommand({}));
|
||||
}
|
||||
|
||||
private parseQueueProperties() {
|
||||
const props = this.queueProperties.split(';');
|
||||
props.forEach(p => {
|
||||
const delimiterPosition = p.indexOf(':');
|
||||
this.queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
|
||||
});
|
||||
}
|
||||
|
||||
private static topicToSqsQueueName(topic: string): string {
|
||||
return topic.replace(/\./g, '_') + '.fifo';
|
||||
}
|
||||
|
||||
private async createQueue(topic: string): Promise<string> {
|
||||
let queueName = AwsSqsTemplate.topicToSqsQueueName(topic);
|
||||
let queueParams: CreateQueueRequest = {
|
||||
QueueName: queueName,
|
||||
Attributes: this.queueAttributes
|
||||
};
|
||||
|
||||
const result = await this.sqsClient.send(new CreateQueueCommand(queueParams));
|
||||
return result.QueueUrl || '';
|
||||
}
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
this.logger.info('Stopping AWS SQS resources...');
|
||||
clearTimeout(this.timer);
|
||||
if (this.sqsClient) {
|
||||
this.logger.info('Stopping AWS SQS client...');
|
||||
try {
|
||||
const _sqsClient = this.sqsClient;
|
||||
// @ts-ignore
|
||||
delete this.sqsClient;
|
||||
_sqsClient.destroy();
|
||||
this.logger.info('AWS SQS client stopped.');
|
||||
} catch (e: any) {
|
||||
this.logger.info('AWS SQS client stop error.');
|
||||
}
|
||||
}
|
||||
this.logger.info('AWS SQS resources stopped.')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
import config from 'config';
|
||||
import { _logger, KafkaJsWinstonLogCreator } from '../config/logger';
|
||||
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
|
||||
import { IQueue } from './queue.models';
|
||||
import {
|
||||
Admin,
|
||||
CompressionTypes,
|
||||
Consumer,
|
||||
Kafka,
|
||||
KafkaConfig,
|
||||
logLevel,
|
||||
Partitioners,
|
||||
Producer,
|
||||
TopicMessages
|
||||
} from 'kafkajs';
|
||||
|
||||
import process, { kill, exit } from 'process';
|
||||
|
||||
export class KafkaTemplate implements IQueue {
|
||||
|
||||
private logger = _logger(`kafkaTemplate`);
|
||||
private replicationFactor = Number(config.get('kafka.replication_factor'));
|
||||
private topicProperties: string = config.get('kafka.topic_properties');
|
||||
private kafkaClientId: string = config.get('kafka.client_id');
|
||||
private acks = Number(config.get('kafka.acks'));
|
||||
private maxBatchSize = Number(config.get('kafka.batch_size'));
|
||||
private linger = Number(config.get('kafka.linger_ms'));
|
||||
private requestTimeout = Number(config.get('kafka.requestTimeout'));
|
||||
private connectionTimeout = Number(config.get('kafka.connectionTimeout'));
|
||||
private compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None;
|
||||
private partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently'));
|
||||
|
||||
private kafkaClient: Kafka;
|
||||
private kafkaAdmin: Admin;
|
||||
private consumer: Consumer;
|
||||
private producer: Producer;
|
||||
private configEntries: any[] = [];
|
||||
private batchMessages: TopicMessages[] = [];
|
||||
private sendLoopInstance: NodeJS.Timeout;
|
||||
|
||||
name = 'Kafka';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers');
|
||||
const requestTopic: string = config.get('request_topic');
|
||||
const useConfluent = config.get('kafka.use_confluent_cloud');
|
||||
|
||||
this.logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
|
||||
this.logger.info('Kafka Requests Topic: %s', requestTopic);
|
||||
|
||||
let kafkaConfig: KafkaConfig = {
|
||||
brokers: kafkaBootstrapServers.split(','),
|
||||
logLevel: logLevel.INFO,
|
||||
logCreator: KafkaJsWinstonLogCreator
|
||||
};
|
||||
|
||||
if (this.kafkaClientId) {
|
||||
kafkaConfig['clientId'] = this.kafkaClientId;
|
||||
} else {
|
||||
this.logger.warn('KAFKA_CLIENT_ID is undefined. Consider to define the env variable KAFKA_CLIENT_ID');
|
||||
}
|
||||
|
||||
kafkaConfig['requestTimeout'] = this.requestTimeout;
|
||||
|
||||
kafkaConfig['connectionTimeout'] = this.connectionTimeout;
|
||||
|
||||
if (useConfluent) {
|
||||
kafkaConfig['sasl'] = {
|
||||
mechanism: config.get('kafka.confluent.sasl.mechanism') as any,
|
||||
username: config.get('kafka.confluent.username'),
|
||||
password: config.get('kafka.confluent.password')
|
||||
};
|
||||
kafkaConfig['ssl'] = true;
|
||||
}
|
||||
|
||||
this.parseTopicProperties();
|
||||
|
||||
this.kafkaClient = new Kafka(kafkaConfig);
|
||||
this.kafkaAdmin = this.kafkaClient.admin();
|
||||
await this.kafkaAdmin.connect();
|
||||
|
||||
let partitions = 1;
|
||||
|
||||
for (let i = 0; i < this.configEntries.length; i++) {
|
||||
let param = this.configEntries[i];
|
||||
if (param.name === 'partitions') {
|
||||
partitions = param.value;
|
||||
this.configEntries.splice(i, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let topics = await this.kafkaAdmin.listTopics();
|
||||
|
||||
if (!topics.includes(requestTopic)) {
|
||||
let createRequestTopicResult = await this.createTopic(requestTopic, partitions);
|
||||
if (createRequestTopicResult) {
|
||||
this.logger.info('Created new topic: %s', requestTopic);
|
||||
}
|
||||
}
|
||||
|
||||
this.consumer = this.kafkaClient.consumer({groupId: 'js-executor-group'});
|
||||
this.producer = this.kafkaClient.producer({createPartitioner: Partitioners.DefaultPartitioner});
|
||||
|
||||
const {CRASH} = this.consumer.events;
|
||||
|
||||
this.consumer.on(CRASH, async (e) => {
|
||||
this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`);
|
||||
if (!e.payload.restart) {
|
||||
this.logger.error('Going to exit due to not retryable error!');
|
||||
kill(process.pid, 'SIGTERM'); //sending signal to myself process to trigger the handler
|
||||
await this.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
const messageProcessor = new JsInvokeMessageProcessor(this);
|
||||
await this.consumer.connect();
|
||||
await this.producer.connect();
|
||||
this.sendLoopWithLinger();
|
||||
await this.consumer.subscribe({topic: requestTopic});
|
||||
|
||||
await this.consumer.run({
|
||||
partitionsConsumedConcurrently: this.partitionsConsumedConcurrently,
|
||||
eachMessage: async ({topic, partition, message}) => {
|
||||
let headers = message.headers;
|
||||
let key = message.key || new Buffer([]);
|
||||
let msg = {
|
||||
key: key.toString('utf8'),
|
||||
data: message.value,
|
||||
headers: {
|
||||
data: headers
|
||||
}
|
||||
};
|
||||
messageProcessor.onJsInvokeMessage(msg);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
|
||||
const message = {
|
||||
topic: responseTopic,
|
||||
messages: [{
|
||||
key: msgKey,
|
||||
value: rawResponse,
|
||||
headers: headers.data
|
||||
}]
|
||||
};
|
||||
|
||||
await this.pushMessageToSendLater(message);
|
||||
}
|
||||
|
||||
private async pushMessageToSendLater(message: TopicMessages) {
|
||||
this.batchMessages.push(message);
|
||||
if (this.batchMessages.length >= this.maxBatchSize) {
|
||||
await this.sendMessagesAsBatch(true);
|
||||
}
|
||||
}
|
||||
|
||||
private async sendMessagesAsBatch(isImmediately = false): Promise<void> {
|
||||
if (this.sendLoopInstance) {
|
||||
clearTimeout(this.sendLoopInstance);
|
||||
}
|
||||
if (this.batchMessages.length > 0) {
|
||||
this.logger.debug('sendMessagesAsBatch, length: [%s], %s', this.batchMessages.length, isImmediately ? 'immediately' : '');
|
||||
const messagesToSend = this.batchMessages;
|
||||
this.batchMessages = [];
|
||||
try {
|
||||
await this.producer.sendBatch({
|
||||
topicMessages: messagesToSend,
|
||||
acks: this.acks,
|
||||
compression: this.compressionType
|
||||
})
|
||||
this.logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length);
|
||||
} catch (err: any) {
|
||||
this.logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length);
|
||||
this.logger.error(err.stack);
|
||||
this.batchMessages = messagesToSend.concat(this.batchMessages);
|
||||
}
|
||||
}
|
||||
this.sendLoopWithLinger();
|
||||
}
|
||||
|
||||
private parseTopicProperties() {
|
||||
const props = this.topicProperties.split(';');
|
||||
props.forEach(p => {
|
||||
const delimiterPosition = p.indexOf(':');
|
||||
this.configEntries.push({
|
||||
name: p.substring(0, delimiterPosition),
|
||||
value: p.substring(delimiterPosition + 1)
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private createTopic(topic: string, partitions: number): Promise<boolean> {
|
||||
return this.kafkaAdmin.createTopics({
|
||||
topics: [{
|
||||
topic: topic,
|
||||
numPartitions: partitions,
|
||||
replicationFactor: this.replicationFactor,
|
||||
configEntries: this.configEntries
|
||||
}]
|
||||
});
|
||||
}
|
||||
|
||||
private sendLoopWithLinger() {
|
||||
if (this.sendLoopInstance) {
|
||||
clearTimeout(this.sendLoopInstance);
|
||||
// } else {
|
||||
// this.logger.debug("Starting new send loop with linger [%s]", this.linger)
|
||||
}
|
||||
this.sendLoopInstance = setTimeout(async () => {
|
||||
await this.sendMessagesAsBatch()
|
||||
}, this.linger);
|
||||
}
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
this.logger.info('Stopping Kafka resources...');
|
||||
|
||||
if (this.kafkaAdmin) {
|
||||
this.logger.info('Stopping Kafka Admin...');
|
||||
const _kafkaAdmin = this.kafkaAdmin;
|
||||
// @ts-ignore
|
||||
delete this.kafkaAdmin;
|
||||
await _kafkaAdmin.disconnect();
|
||||
this.logger.info('Kafka Admin stopped.');
|
||||
}
|
||||
|
||||
if (this.consumer) {
|
||||
this.logger.info('Stopping Kafka Consumer...');
|
||||
try {
|
||||
const _consumer = this.consumer;
|
||||
// @ts-ignore
|
||||
delete this.consumer;
|
||||
await _consumer.disconnect();
|
||||
this.logger.info('Kafka Consumer stopped.');
|
||||
await this.disconnectProducer();
|
||||
} catch (e: any) {
|
||||
this.logger.info('Kafka Consumer stop error.');
|
||||
await this.disconnectProducer();
|
||||
}
|
||||
}
|
||||
this.logger.info('Kafka resources stopped.');
|
||||
exit(0); //same as in version before
|
||||
}
|
||||
|
||||
private async disconnectProducer(): Promise<void> {
|
||||
if (this.producer) {
|
||||
this.logger.info('Stopping Kafka Producer...');
|
||||
try {
|
||||
this.logger.info('Stopping loop...');
|
||||
clearTimeout(this.sendLoopInstance);
|
||||
await this.sendMessagesAsBatch();
|
||||
const _producer = this.producer;
|
||||
// @ts-ignore
|
||||
delete this.producer;
|
||||
await _producer.disconnect();
|
||||
this.logger.info('Kafka Producer stopped.');
|
||||
} catch (e) {
|
||||
this.logger.info('Kafka Producer stop error.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
import config from 'config';
|
||||
import { _logger } from '../config/logger';
|
||||
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
|
||||
import { PubSub } from '@google-cloud/pubsub';
|
||||
import { IQueue } from './queue.models';
|
||||
import { Message } from '@google-cloud/pubsub/build/src/subscriber';
|
||||
|
||||
export class PubSubTemplate implements IQueue {
|
||||
|
||||
private logger = _logger(`pubSubTemplate`);
|
||||
private projectId: string = config.get('pubsub.project_id');
|
||||
private credentials = JSON.parse(config.get('pubsub.service_account'));
|
||||
private requestTopic: string = config.get('request_topic');
|
||||
private queueProperties: string = config.get('pubsub.queue_properties');
|
||||
|
||||
private pubSubClient: PubSub;
|
||||
private queueProps: { [n: string]: string } = {};
|
||||
private topics: string[] = [];
|
||||
private subscriptions: string[] = [];
|
||||
|
||||
name = 'Pub/Sub';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
async init() {
|
||||
this.pubSubClient = new PubSub({
|
||||
projectId: this.projectId,
|
||||
credentials: this.credentials
|
||||
});
|
||||
|
||||
this.parseQueueProperties();
|
||||
|
||||
const topicList = await this.pubSubClient.getTopics();
|
||||
|
||||
if (topicList) {
|
||||
topicList[0].forEach(topic => {
|
||||
this.topics.push(PubSubTemplate.getName(topic.name));
|
||||
});
|
||||
}
|
||||
|
||||
const subscriptionList = await this.pubSubClient.getSubscriptions();
|
||||
|
||||
if (subscriptionList) {
|
||||
topicList[0].forEach(sub => {
|
||||
this.subscriptions.push(PubSubTemplate.getName(sub.name));
|
||||
});
|
||||
}
|
||||
|
||||
if (!(this.subscriptions.includes(this.requestTopic) && this.topics.includes(this.requestTopic))) {
|
||||
await this.createTopic(this.requestTopic);
|
||||
await this.createSubscription(this.requestTopic);
|
||||
}
|
||||
|
||||
const subscription = this.pubSubClient.subscription(this.requestTopic);
|
||||
|
||||
const messageProcessor = new JsInvokeMessageProcessor(this);
|
||||
|
||||
const messageHandler = (message: Message) => {
|
||||
messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8')));
|
||||
message.ack();
|
||||
};
|
||||
|
||||
subscription.on('message', messageHandler);
|
||||
}
|
||||
|
||||
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
|
||||
if (!(this.subscriptions.includes(responseTopic) && this.topics.includes(this.requestTopic))) {
|
||||
await this.createTopic(this.requestTopic);
|
||||
await this.createSubscription(this.requestTopic);
|
||||
}
|
||||
|
||||
let data = JSON.stringify(
|
||||
{
|
||||
key: msgKey,
|
||||
data: [...rawResponse],
|
||||
headers: headers
|
||||
});
|
||||
let dataBuffer = Buffer.from(data);
|
||||
return this.pubSubClient.topic(responseTopic).publishMessage({data: dataBuffer});
|
||||
}
|
||||
|
||||
private parseQueueProperties() {
|
||||
const props = this.queueProperties.split(';');
|
||||
props.forEach(p => {
|
||||
const delimiterPosition = p.indexOf(':');
|
||||
this.queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
|
||||
});
|
||||
}
|
||||
|
||||
private static getName(fullName: string): string {
|
||||
const delimiterPosition = fullName.lastIndexOf('/');
|
||||
return fullName.substring(delimiterPosition + 1);
|
||||
}
|
||||
|
||||
private async createTopic(topic: string) {
|
||||
if (!this.topics.includes(topic)) {
|
||||
try {
|
||||
await this.pubSubClient.createTopic(topic);
|
||||
this.logger.info('Created new Pub/Sub topic: %s', topic);
|
||||
} catch (e) {
|
||||
this.logger.info('Pub/Sub topic already exists');
|
||||
}
|
||||
this.topics.push(topic);
|
||||
}
|
||||
}
|
||||
|
||||
private async createSubscription(topic: string) {
|
||||
if (!this.subscriptions.includes(topic)) {
|
||||
try {
|
||||
await this.pubSubClient.createSubscription(topic, topic, {
|
||||
topic: topic,
|
||||
name: topic,
|
||||
ackDeadlineSeconds: Number(this.queueProps['ackDeadlineInSec']),
|
||||
messageRetentionDuration: {
|
||||
seconds: this.queueProps['messageRetentionInSec']
|
||||
}
|
||||
});
|
||||
this.logger.info('Created new Pub/Sub subscription: %s', topic);
|
||||
} catch (e) {
|
||||
this.logger.info('Pub/Sub subscription already exists.');
|
||||
}
|
||||
|
||||
this.subscriptions.push(topic);
|
||||
}
|
||||
}
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
this.logger.info('Stopping Pub/Sub resources...');
|
||||
if (this.pubSubClient) {
|
||||
this.logger.info('Stopping Pub/Sub client...');
|
||||
try {
|
||||
const _pubSubClient = this.pubSubClient;
|
||||
// @ts-ignore
|
||||
delete this.pubSubClient;
|
||||
await _pubSubClient.close();
|
||||
this.logger.info('Pub/Sub client stopped.');
|
||||
} catch (e) {
|
||||
this.logger.info('Pub/Sub client stop error.');
|
||||
}
|
||||
}
|
||||
this.logger.info('Pub/Sub resources stopped.');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
export interface IQueue {
|
||||
name: string;
|
||||
init(): Promise<void>;
|
||||
send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any>;
|
||||
destroy(): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
import config from 'config';
|
||||
import { _logger } from '../config/logger';
|
||||
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
|
||||
import { IQueue } from './queue.models';
|
||||
import amqp, { ConfirmChannel, Connection } from 'amqplib';
|
||||
import { Options, Replies } from 'amqplib/properties';
|
||||
|
||||
export class RabbitMqTemplate implements IQueue {
|
||||
|
||||
private logger = _logger(`rabbitmqTemplate`);
|
||||
private requestTopic: string = config.get('request_topic');
|
||||
private host = config.get('rabbitmq.host');
|
||||
private port = config.get('rabbitmq.port');
|
||||
private vhost = config.get('rabbitmq.virtual_host');
|
||||
private username = config.get('rabbitmq.username');
|
||||
private password = config.get('rabbitmq.password');
|
||||
private queueProperties: string = config.get('rabbitmq.queue_properties');
|
||||
|
||||
private queueOptions: Options.AssertQueue = {
|
||||
durable: false,
|
||||
exclusive: false,
|
||||
autoDelete: false
|
||||
};
|
||||
private connection: Connection;
|
||||
private channel: ConfirmChannel;
|
||||
private topics: string[] = [];
|
||||
|
||||
name = 'RabbitMQ';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`;
|
||||
this.connection = await amqp.connect(url);
|
||||
this.channel = await this.connection.createConfirmChannel();
|
||||
|
||||
this.parseQueueProperties();
|
||||
|
||||
await this.createQueue(this.requestTopic);
|
||||
|
||||
const messageProcessor = new JsInvokeMessageProcessor(this);
|
||||
|
||||
await this.channel.consume(this.requestTopic, (message) => {
|
||||
if (message) {
|
||||
messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8')));
|
||||
this.channel.ack(message);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
|
||||
|
||||
if (!this.topics.includes(responseTopic)) {
|
||||
await this.createQueue(responseTopic);
|
||||
this.topics.push(responseTopic);
|
||||
}
|
||||
|
||||
let data = JSON.stringify(
|
||||
{
|
||||
key: msgKey,
|
||||
data: [...rawResponse],
|
||||
headers: headers
|
||||
});
|
||||
let dataBuffer = Buffer.from(data);
|
||||
this.channel.sendToQueue(responseTopic, dataBuffer);
|
||||
return this.channel.waitForConfirms()
|
||||
}
|
||||
|
||||
private parseQueueProperties() {
|
||||
let args: { [n: string]: number } = {};
|
||||
const props = this.queueProperties.split(';');
|
||||
props.forEach(p => {
|
||||
const delimiterPosition = p.indexOf(':');
|
||||
args[p.substring(0, delimiterPosition)] = Number(p.substring(delimiterPosition + 1));
|
||||
});
|
||||
this.queueOptions['arguments'] = args;
|
||||
}
|
||||
|
||||
private async createQueue(topic: string): Promise<Replies.AssertQueue> {
|
||||
return this.channel.assertQueue(topic, this.queueOptions);
|
||||
}
|
||||
|
||||
async destroy() {
|
||||
this.logger.info('Stopping RabbitMQ resources...');
|
||||
|
||||
if (this.channel) {
|
||||
this.logger.info('Stopping RabbitMQ chanel...');
|
||||
const _channel = this.channel;
|
||||
// @ts-ignore
|
||||
delete this.channel;
|
||||
await _channel.close();
|
||||
this.logger.info('RabbitMQ chanel stopped');
|
||||
}
|
||||
|
||||
if (this.connection) {
|
||||
this.logger.info('Stopping RabbitMQ connection...')
|
||||
try {
|
||||
const _connection = this.connection;
|
||||
// @ts-ignore
|
||||
delete this.connection;
|
||||
await _connection.close();
|
||||
this.logger.info('RabbitMQ client connection.');
|
||||
} catch (e) {
|
||||
this.logger.info('RabbitMQ connection stop error.');
|
||||
}
|
||||
}
|
||||
this.logger.info('RabbitMQ resources stopped.')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
///
|
||||
/// Copyright © 2016-2023 The Thingsboard Authors
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
|
||||
import config from 'config';
|
||||
import { _logger } from '../config/logger';
|
||||
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
|
||||
import { IQueue } from './queue.models';
|
||||
import {
|
||||
CreateQueueOptions,
|
||||
ProcessErrorArgs,
|
||||
ServiceBusAdministrationClient,
|
||||
ServiceBusClient,
|
||||
ServiceBusReceivedMessage,
|
||||
ServiceBusReceiver,
|
||||
ServiceBusSender
|
||||
} from '@azure/service-bus';
|
||||
|
||||
export class ServiceBusTemplate implements IQueue {
|
||||
|
||||
private logger = _logger(`serviceBusTemplate`);
|
||||
private requestTopic: string = config.get('request_topic');
|
||||
private namespaceName = config.get('service_bus.namespace_name');
|
||||
private sasKeyName = config.get('service_bus.sas_key_name');
|
||||
private sasKey = config.get('service_bus.sas_key');
|
||||
private queueProperties: string = config.get('service_bus.queue_properties');
|
||||
|
||||
private sbClient: ServiceBusClient;
|
||||
private serviceBusService: ServiceBusAdministrationClient;
|
||||
private queueOptions: CreateQueueOptions = {};
|
||||
private queues: string[] = [];
|
||||
private receiver: ServiceBusReceiver;
|
||||
private senderMap = new Map<string, ServiceBusSender>();
|
||||
|
||||
name = 'Azure Service Bus';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
async init() {
|
||||
const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`;
|
||||
this.sbClient = new ServiceBusClient(connectionString)
|
||||
this.serviceBusService = new ServiceBusAdministrationClient(connectionString);
|
||||
|
||||
this.parseQueueProperties();
|
||||
|
||||
const listQueues = await this.serviceBusService.listQueues();
|
||||
for await (const queue of listQueues) {
|
||||
this.queues.push(queue.name);
|
||||
}
|
||||
|
||||
if (!this.queues.includes(this.requestTopic)) {
|
||||
await this.createQueueIfNotExist(this.requestTopic);
|
||||
this.queues.push(this.requestTopic);
|
||||
}
|
||||
|
||||
this.receiver = this.sbClient.createReceiver(this.requestTopic, {receiveMode: 'peekLock'});
|
||||
|
||||
const messageProcessor = new JsInvokeMessageProcessor(this);
|
||||
|
||||
const messageHandler = async (message: ServiceBusReceivedMessage) => {
|
||||
if (message) {
|
||||
messageProcessor.onJsInvokeMessage(message.body);
|
||||
await this.receiver.completeMessage(message);
|
||||
}
|
||||
};
|
||||
const errorHandler = async (error: ProcessErrorArgs) => {
|
||||
this.logger.error('Failed to receive message from queue.', error);
|
||||
};
|
||||
this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler})
|
||||
}
|
||||
|
||||
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
|
||||
if (!this.queues.includes(this.requestTopic)) {
|
||||
await this.createQueueIfNotExist(this.requestTopic);
|
||||
this.queues.push(this.requestTopic);
|
||||
}
|
||||
|
||||
let customSender = this.senderMap.get(responseTopic);
|
||||
|
||||
if (!customSender) {
|
||||
customSender = this.sbClient.createSender(responseTopic);
|
||||
this.senderMap.set(responseTopic, customSender);
|
||||
}
|
||||
|
||||
let data = {
|
||||
key: msgKey,
|
||||
data: [...rawResponse],
|
||||
headers: headers
|
||||
};
|
||||
|
||||
return customSender.sendMessages({body: data});
|
||||
}
|
||||
|
||||
private parseQueueProperties() {
|
||||
let properties: { [n: string]: string } = {};
|
||||
const props = this.queueProperties.split(';');
|
||||
props.forEach(p => {
|
||||
const delimiterPosition = p.indexOf(':');
|
||||
properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
|
||||
});
|
||||
this.queueOptions = {
|
||||
requiresDuplicateDetection: false,
|
||||
maxSizeInMegabytes: Number(properties['maxSizeInMb']),
|
||||
defaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`,
|
||||
lockDuration: `PT${properties['lockDurationInSec']}S`
|
||||
};
|
||||
}
|
||||
|
||||
private async createQueueIfNotExist(topic: string) {
|
||||
try {
|
||||
await this.serviceBusService.createQueue(topic, this.queueOptions)
|
||||
} catch (err: any) {
|
||||
if (err && err.code !== "MessageEntityAlreadyExistsError") {
|
||||
throw new Error(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async destroy() {
|
||||
this.logger.info('Stopping Azure Service Bus resources...')
|
||||
if (this.receiver) {
|
||||
this.logger.info('Stopping Service Bus Receiver...');
|
||||
try {
|
||||
const _receiver = this.receiver;
|
||||
// @ts-ignore
|
||||
delete this.receiver;
|
||||
await _receiver.close();
|
||||
this.logger.info('Service Bus Receiver stopped.');
|
||||
} catch (e) {
|
||||
this.logger.info('Service Bus Receiver stop error.');
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Stopping Service Bus Senders...');
|
||||
const senders: Promise<void>[] = [];
|
||||
this.senderMap.forEach((sender) => {
|
||||
senders.push(sender.close());
|
||||
});
|
||||
this.senderMap.clear();
|
||||
try {
|
||||
await Promise.all(senders);
|
||||
this.logger.info('Service Bus Senders stopped.');
|
||||
} catch (e) {
|
||||
this.logger.info('Service Bus Senders stop error.');
|
||||
}
|
||||
|
||||
if (this.sbClient) {
|
||||
this.logger.info('Stopping Service Bus Client...');
|
||||
try {
|
||||
const _sbClient = this.sbClient;
|
||||
// @ts-ignore
|
||||
delete this.sbClient;
|
||||
await _sbClient.close();
|
||||
this.logger.info('Service Bus Client stopped.');
|
||||
} catch (e) {
|
||||
this.logger.info('Service Bus Client stop error.');
|
||||
}
|
||||
}
|
||||
this.logger.info('Azure Service Bus resources stopped.')
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user