[Feat] Allow AWS SECRETS MANAGER instead of storing AES Encrypted in db

packages/components/package.json

@@ -24,6 +24,7 @@
         "@aws-sdk/client-bedrock-runtime": "3.422.0",
         "@aws-sdk/client-dynamodb": "^3.360.0",
         "@aws-sdk/client-s3": "^3.427.0",
+        "@aws-sdk/client-secrets-manager": "^3.699.0",
         "@datastax/astra-db-ts": "1.5.0",
         "@dqbd/tiktoken": "^1.0.7",
         "@e2b/code-interpreter": "^0.0.5",

packages/components/src/utils.ts

@@ -9,11 +9,21 @@ import { ICommonObject, IDatabaseEntity, IDocument, IMessage, INodeData, IVariab
 import { AES, enc } from 'crypto-js'
 import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
 import { getFileFromStorage } from './storageUtils'
+import { GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'
+import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager'
 
 export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
 export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
 export const FLOWISE_CHATID = 'flowise_chatId'
+const USE_AWS_SECRETS_MANAGER = process.env.USE_AWS_SECRETS_MANAGER === 'true'
+const AWS_REGION = process.env.AWS_REGION || 'us-east-1' // Default region if not provided
 
+let secretsManagerClient: SecretsManagerClient | null = null
+if (USE_AWS_SECRETS_MANAGER) {
+    secretsManagerClient = new SecretsManagerClient({ region: AWS_REGION })
+}
+const CACHE_CREDENTIALS = process.env.CACHE_CREDENTIALS === 'true'
+let credentialsCache = new Map<string, string>()
 /*
  * List of dependencies allowed to be import in @flowiseai/nodevm
  */
@@ -502,10 +512,40 @@ const getEncryptionKey = async (): Promise<string> => {
  * @returns {Promise<ICommonObject>}
  */
 const decryptCredentialData = async (encryptedData: string): Promise<ICommonObject> => {
-    const encryptKey = await getEncryptionKey()
-    const decryptedData = AES.decrypt(encryptedData, encryptKey)
+    if (credentialsCache.has(encryptedData)) {
+        return JSON.parse(credentialsCache.get(encryptedData) ?? '{}') as ICommonObject
+    }
+
+    let decryptedDataStr: string
+
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        try {
+            const command = new GetSecretValueCommand({ SecretId: encryptedData })
+            const response = await secretsManagerClient.send(command)
+
+            if (response.SecretString) {
+                const secretObj = JSON.parse(response.SecretString)
+                decryptedDataStr = JSON.stringify(secretObj)
+            } else {
+                throw new Error('Failed to retrieve secret value.')
+            }
+        } catch (error) {
+            console.error(error)
+            throw new Error('Credentials could not be decrypted.')
+        }
+    } else {
+        // Fallback to existing code
+        const encryptKey = await getEncryptionKey()
+        const decryptedData = AES.decrypt(encryptedData, encryptKey)
+        decryptedDataStr = decryptedData.toString(enc.Utf8)
+    }
+
+    if (!decryptedDataStr) return {}
     try {
-        return JSON.parse(decryptedData.toString(enc.Utf8))
+        if (CACHE_CREDENTIALS) {
+            credentialsCache.set(encryptedData, decryptedDataStr)
+        }
+        return JSON.parse(decryptedDataStr)
     } catch (e) {
         console.error(e)
         throw new Error('Credentials could not be decrypted.')

packages/server/.env.example

@@ -71,3 +71,8 @@ PORT=3000
 # GLOBAL_AGENT_HTTP_PROXY=CorporateHttpProxyUrl
 # GLOBAL_AGENT_HTTPS_PROXY=CorporateHttpsProxyUrl
 # GLOBAL_AGENT_NO_PROXY=ExceptionHostsToBypassProxyIfNeeded
+##AWS Secrets Manager
+# AWS_ACCESS_KEY_ID=<your-access-key>
+# AWS_SECRET_ACCESS_KEY=<your-secret-key>
+# USE_AWS_SECRETS_MANAGER=true
+# CACHE_CREDENTIALS=true

packages/server/package.json

@@ -54,8 +54,10 @@
     },
     "license": "SEE LICENSE IN LICENSE.md",
     "dependencies": {
+        "@aws-sdk/client-secrets-manager": "^3.699.0",
         "@oclif/core": "^1.13.10",
         "@opentelemetry/api": "^1.3.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.52.0",
         "@opentelemetry/core": "1.27.0",
         "@opentelemetry/exporter-metrics-otlp-grpc": "0.54.0",
         "@opentelemetry/exporter-metrics-otlp-http": "0.54.0",
@@ -65,10 +67,9 @@
         "@opentelemetry/exporter-trace-otlp-proto": "0.54.0",
         "@opentelemetry/resources": "1.27.0",
         "@opentelemetry/sdk-metrics": "1.27.0",
+        "@opentelemetry/sdk-node": "^0.54.0",
         "@opentelemetry/sdk-trace-base": "1.27.0",
         "@opentelemetry/semantic-conventions": "1.27.0",
-        "@opentelemetry/auto-instrumentations-node": "^0.52.0",
-        "@opentelemetry/sdk-node": "^0.54.0",
         "@types/lodash": "^4.14.202",
         "@types/uuid": "^9.0.7",
         "async-mutex": "^0.4.0",
@@ -82,6 +83,7 @@
         "express-rate-limit": "^6.9.0",
         "flowise-components": "workspace:^",
         "flowise-ui": "workspace:^",
+        "global-agent": "^3.0.0",
         "http-errors": "^2.0.0",
         "http-status-codes": "^2.3.0",
         "langchainhub": "^0.0.11",
@@ -100,8 +102,7 @@
         "sqlite3": "^5.1.6",
         "typeorm": "^0.3.6",
         "uuid": "^9.0.1",
-        "winston": "^3.9.0",
-        "global-agent": "^3.0.0"
+        "winston": "^3.9.0"
     },
     "devDependencies": {
         "@types/content-disposition": "0.5.8",

packages/server/src/utils/index.ts

@@ -49,12 +49,23 @@ import { DocumentStore } from '../database/entities/DocumentStore'
 import { DocumentStoreFileChunk } from '../database/entities/DocumentStoreFileChunk'
 import { InternalFlowiseError } from '../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
-
+import { CreateSecretCommand, GetSecretValueCommand, PutSecretValueCommand } from '@aws-sdk/client-secrets-manager'
+import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager'
 const QUESTION_VAR_PREFIX = 'question'
 const FILE_ATTACHMENT_PREFIX = 'file_attachment'
 const CHAT_HISTORY_VAR_PREFIX = 'chat_history'
 const REDACTED_CREDENTIAL_VALUE = '_FLOWISE_BLANK_07167752-1a71-43b1-bf8f-4f32252165db'
 
+const USE_AWS_SECRETS_MANAGER = process.env.USE_AWS_SECRETS_MANAGER === 'true'
+const AWS_REGION = process.env.AWS_REGION || 'us-east-1' // Default region if not provided
+
+let secretsManagerClient: SecretsManagerClient | null = null
+if (USE_AWS_SECRETS_MANAGER) {
+    secretsManagerClient = new SecretsManagerClient({ region: AWS_REGION })
+}
+const CACHE_CREDENTIALS = process.env.CACHE_CREDENTIALS === 'true'
+let credentialsCache = new Map<string, string>()
+
 export const databaseEntities: IDatabaseEntity = {
     ChatFlow: ChatFlow,
     ChatMessage: ChatMessage,
@@ -1345,14 +1356,6 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
     return isChatOrLLMsExist && isValidChainOrAgent && !isOutputParserExist
 }
 
-/**
- * Generate an encryption key
- * @returns {string}
- */
-export const generateEncryptKey = (): string => {
-    return randomBytes(24).toString('base64')
-}
-
 /**
  * Returns the encryption key
  * @returns {Promise<string>}
@@ -1374,17 +1377,49 @@ export const getEncryptionKey = async (): Promise<string> => {
 }
 
 /**
- * Encrypt credential data
+ * Encrypt credential data using AWS Secrets Manager if USE_AWS_SECRETS_MANAGER is set to true.
  * @param {ICredentialDataDecrypted} plainDataObj
  * @returns {Promise<string>}
  */
 export const encryptCredentialData = async (plainDataObj: ICredentialDataDecrypted): Promise<string> => {
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        const secretName = `FlowiseCredential_${randomBytes(12).toString('hex')}`
+
+        logger.info(`[server]: Upserting AWS Secret: ${secretName}`)
+
+        const secretString = JSON.stringify({ ...plainDataObj })
+
+        try {
+            // Try to update the secret if it exists
+            const putCommand = new PutSecretValueCommand({
+                SecretId: secretName,
+                SecretString: secretString
+            })
+            await secretsManagerClient.send(putCommand)
+        } catch (error: any) {
+            if (error.name === 'ResourceNotFoundException') {
+                // Secret doesn't exist, so create it
+                const createCommand = new CreateSecretCommand({
+                    Name: secretName,
+                    SecretString: secretString
+                })
+                await secretsManagerClient.send(createCommand)
+            } else {
+                // Rethrow any other errors
+                throw error
+            }
+        }
+        return secretName
+    }
+
     const encryptKey = await getEncryptionKey()
+
+    // Fallback to existing code
     return AES.encrypt(JSON.stringify(plainDataObj), encryptKey).toString()
 }
 
 /**
- * Decrypt credential data
+ * Decrypt credential data using AWS Secrets Manager if USE_AWS_SECRETS_MANAGER is set to true.
  * @param {string} encryptedData
  * @param {string} componentCredentialName
  * @param {IComponentCredentials} componentCredentials
@@ -1395,22 +1430,58 @@ export const decryptCredentialData = async (
     componentCredentialName?: string,
     componentCredentials?: IComponentCredentials
 ): Promise<ICredentialDataDecrypted> => {
-    const encryptKey = await getEncryptionKey()
-    const decryptedData = AES.decrypt(encryptedData, encryptKey)
-    const decryptedDataStr = decryptedData.toString(enc.Utf8)
+    if (credentialsCache.has(encryptedData)) {
+        return JSON.parse(credentialsCache.get(encryptedData) ?? '{}') as ICredentialDataDecrypted
+    }
+    let decryptedDataStr: string
+
+    if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
+        try {
+            logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
+            const command = new GetSecretValueCommand({ SecretId: encryptedData })
+            const response = await secretsManagerClient.send(command)
+
+            if (response.SecretString) {
+                const secretObj = JSON.parse(response.SecretString)
+                decryptedDataStr = JSON.stringify(secretObj)
+            } else {
+                throw new Error('Failed to retrieve secret value.')
+            }
+        } catch (error) {
+            console.error(error)
+            throw new Error('Failed to decrypt credential data.')
+        }
+    } else {
+        // Fallback to existing code
+        const encryptKey = await getEncryptionKey()
+        const decryptedData = AES.decrypt(encryptedData, encryptKey)
+        decryptedDataStr = decryptedData.toString(enc.Utf8)
+    }
+
     if (!decryptedDataStr) return {}
     try {
         if (componentCredentialName && componentCredentials) {
-            const plainDataObj = JSON.parse(decryptedData.toString(enc.Utf8))
+            const plainDataObj = JSON.parse(decryptedDataStr)
             return redactCredentialWithPasswordType(componentCredentialName, plainDataObj, componentCredentials)
         }
-        return JSON.parse(decryptedData.toString(enc.Utf8))
+        if (CACHE_CREDENTIALS) {
+            credentialsCache.set(encryptedData, decryptedDataStr)
+        }
+        return JSON.parse(decryptedDataStr)
     } catch (e) {
         console.error(e)
         return {}
     }
 }
 
+/**
+ * Generate an encryption key
+ * @returns {string}
+ */
+export const generateEncryptKey = (): string => {
+    return randomBytes(24).toString('base64')
+}
+
 /**
  * Transform ICredentialBody from req to Credential entity
  * @param {ICredentialReqBody} body