first-fix

msa/js-executor/config/custom-environment-variables.yml

@@ -0,0 +1,84 @@
+#
+# Copyright © 2016-2023 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
+request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC"
+http_port: "HTTP_PORT"      # /livenessProbe
+
+js:
+  response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS"
+  max_result_size: "JS_MAX_RESULT_SIZE"
+
+kafka:
+  bootstrap:
+    # Kafka Bootstrap Servers
+    servers: "TB_KAFKA_SERVERS"
+  replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR"
+  acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge
+  batch_size: "TB_KAFKA_BATCH_SIZE" # for producer
+  linger_ms: "TB_KAFKA_LINGER_MS" # for producer
+  partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency
+  requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS"
+  connectionTimeout: "TB_KAFKA_CONNECTION_TIMEOUT_MS"
+  compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed
+  topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES"
+  use_confluent_cloud: "TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD"
+  client_id: "KAFKA_CLIENT_ID" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh
+  confluent:
+    sasl:
+      mechanism: "TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM"
+    username: "TB_QUEUE_KAFKA_CONFLUENT_USERNAME"
+    password: "TB_QUEUE_KAFKA_CONFLUENT_PASSWORD"
+
+pubsub:
+  project_id: "TB_QUEUE_PUBSUB_PROJECT_ID"
+  service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT"
+  queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES"
+
+aws_sqs:
+  access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID"
+  secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY"
+  region: "TB_QUEUE_AWS_SQS_REGION"
+  queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES"
+
+rabbitmq:
+  host: "TB_QUEUE_RABBIT_MQ_HOST"
+  port: "TB_QUEUE_RABBIT_MQ_PORT"
+  virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST"
+  username: "TB_QUEUE_RABBIT_MQ_USERNAME"
+  password: "TB_QUEUE_RABBIT_MQ_PASSWORD"
+  queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES"
+
+service_bus:
+  namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME"
+  sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME"
+  sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY"
+  max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES"
+  queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES"
+
+logger:
+  level: "LOGGER_LEVEL"
+  path: "LOG_FOLDER"
+  filename: "LOGGER_FILENAME"
+
+script:
+  use_sandbox: "SCRIPT_USE_SANDBOX"
+  memory_usage_trace_frequency: "MEMORY_USAGE_TRACE_FREQUENCY"
+  stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY"
+  script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY"
+  max_active_scripts: "MAX_ACTIVE_SCRIPTS"
+  slow_query_log_ms: "SLOW_QUERY_LOG_MS" #1.123456
+  slow_query_log_body: "SLOW_QUERY_LOG_BODY" #true or false

msa/js-executor/config/default.yml

@@ -0,0 +1,73 @@
+#
+# Copyright © 2016-2023 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+queue_type: "kafka"
+request_topic: "js_eval.requests"
+http_port: "8888" # /livenessProbe
+
+js:
+  response_poll_interval: "25"
+  max_result_size: "300000"
+
+kafka:
+  bootstrap:
+    # Kafka Bootstrap Servers
+    servers: "localhost:9092"
+  replication_factor: "1"
+  acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge
+  batch_size: "128" # for producer
+  linger_ms: "5" # for producer
+  partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency
+  requestTimeout: "30000" # The default value in kafkajs is: 30000
+  connectionTimeout: "1000" # The default value in kafkajs is: 1000
+  compression: "gzip" # gzip or uncompressed
+  topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1"
+  use_confluent_cloud: false
+  client_id: "kafkajs" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh
+  confluent:
+    sasl:
+      mechanism: "PLAIN"
+
+pubsub:
+  queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800"
+
+aws_sqs:
+  queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800"
+
+rabbitmq:
+  host: "localhost"
+  port: "5672"
+  virtual_host: "/"
+  username: "admin"
+  password: "password"
+  queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000"
+
+service_bus:
+  queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800"
+
+logger:
+  level: "info"
+  path: "logs"
+  filename: "tb-js-executor-%DATE%.log"
+
+script:
+  use_sandbox: "true"
+  memory_usage_trace_frequency: "1000"
+  script_body_trace_frequency: "10000"
+  stat_print_frequency: "10000"
+  max_active_scripts: "1000"
+  slow_query_log_ms: "5.000000" #millis
+  slow_query_log_body: "false"

msa/js-executor/config/logger.ts

@@ -0,0 +1,104 @@
+///
+/// Copyright © 2016-2023 The Thingsboard Authors
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+
+import config from 'config';
+import path from 'path';
+import DailyRotateFile from 'winston-daily-rotate-file';
+
+import { LogEntry, logLevel } from 'kafkajs';
+import { createLogger, format, transports } from 'winston';
+import * as Transport from 'winston-transport';
+
+const { combine, timestamp, label, printf, splat } = format;
+
+const toWinstonLogLevel = (level: logLevel): string => {
+    switch(level) {
+        case logLevel.ERROR:
+        case logLevel.NOTHING:
+            return 'error'
+        case logLevel.WARN:
+            return 'warn'
+        case logLevel.INFO:
+            return 'info'
+        case logLevel.DEBUG:
+            return 'debug'
+    }
+}
+
+const loggerTransports: Array<Transport> = [];
+
+if (process.env.NODE_ENV !== 'production' || process.env.DOCKER_MODE === 'true') {
+    loggerTransports.push(new transports.Console({
+        handleExceptions: true
+    }));
+} else {
+    const filename = path.join(config.get('logger.path'), config.get('logger.filename'));
+    const transport = new (DailyRotateFile)({
+        filename: filename,
+        datePattern: 'YYYY-MM-DD-HH',
+        zippedArchive: true,
+        maxSize: '20m',
+        maxFiles: '14d',
+        handleExceptions: true
+    });
+    loggerTransports.push(transport);
+}
+
+const tbFormat = printf(info => {
+    return `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`;
+});
+
+export function _logger(moduleLabel: string) {
+    return createLogger({
+        level: config.get('logger.level'),
+        format:combine(
+            splat(),
+            label({ label: moduleLabel }),
+            timestamp({format: 'YYYY-MM-DD HH:mm:ss,SSS'}),
+            tbFormat
+        ),
+        transports: loggerTransports
+    });
+}
+
+export function KafkaJsWinstonLogCreator(logLevel: logLevel): (entry: LogEntry) => void {
+    const logger = createLogger({
+        level: toWinstonLogLevel(logLevel),
+        format:combine(
+            splat(),
+            label({ label: 'kafkajs' }),
+            timestamp({format: 'YYYY-MM-DD HH:mm:ss,SSS'}),
+            printf(info => {
+                var res = `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`;
+                if (info.extra) {
+                    res +=`: ${JSON.stringify(info.extra)}`;
+                }
+                return res;
+              }
+            )
+        ),
+        transports: loggerTransports
+    });
+
+    return ({ namespace, level, label, log }) => {
+        const { message, ...extra } = log;
+        logger.log({
+            level: toWinstonLogLevel(level),
+            message,
+            extra,
+        });
+    }
+}

msa/js-executor/config/tb-js-executor.conf

@@ -0,0 +1,19 @@
+#
+# Copyright © 2016-2023 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export NODE_CONFIG_DIR=${pkg.installFolder}/conf
+export LOG_FOLDER=${pkg.logFolder}
+export NODE_ENV=production