oliver/awesome-n8n-templates
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change to json formatted text and reformat all .txt to .json

Browse Source
This commit is contained in:
rival14
2025-06-22 00:27:45 +07:00
parent 71f3e50c73
commit b068ce2ab8
584 changed files with 189530 additions and 196635 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
AI_Research_RAG_and_Data_Analysis/Analyze tradingview.com charts with Chrome extension, N8N and OpenAI.json Normal file
View File

@@ -0,0 +1,131 @@
{
"id": "Q8On8rR6BkmPzDUd",
"meta": {
"instanceId": "f57770b08f6a574802832e927ed1b0063c627ffc5b95965abf0d4a7396150138"
},
"name": "chrome extension backend with AI",
"tags": [],
"nodes": [
{
"id": "0f38fe62-36d9-43da-a992-a3981377e89e",
"name": "Webhook",
"type": "n8n-nodes-base.webhook",
"position": [
-220,
-20
],
"webhookId": "e9a97dd5-f1e7-4d5b-a6f1-be5f0c9eb96c",
"parameters": {
"path": "e9a97dd5-f1e7-4d5b-a6f1-be5f0c9eb96c",
"options": {},
"httpMethod": "POST",
"responseMode": "responseNode"
},
"typeVersion": 2
},
{
"id": "83959562-edf5-4d37-bd11-47186c6a31c7",
"name": "OpenAI",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
-40,
-20
],
"parameters": {
"text": "You are an expert financial analyst tasked with providing an advanced technical analyses of a stock or crypto currency chart provided. Your analysis will be based on various technical indicators and will provide simple insights for novice traders. Just explain to traders were you expect the market is moving. Also warn them this is not a binding advice. Make sure to explain everything in infant language.",
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini",
"cachedResultName": "GPT-4O-MINI"
},
"options": {},
"resource": "image",
"inputType": "base64",
"operation": "analyze"
},
"credentials": {
"openAiApi": {
"id": "8MS1muoK4z86fxUs",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
},
{
"id": "c6f1f833-7ba3-49c5-86df-f586e6bb5975",
"name": "Respond to Webhook",
"type": "n8n-nodes-base.respondToWebhook",
"position": [
140,
-20
],
"parameters": {
"options": {},
"respondWith": "text",
"responseBody": "={{ $json.content }}"
},
"typeVersion": 1.1
},
{
"id": "e3a38a76-283b-4567-a8da-315ef1e2bc4f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-260,
-140
],
"parameters": {
"width": 620,
"height": 300,
"content": "## N8N en OpenAI image analyser"
},
"typeVersion": 1
},
{
"id": "8e7e26db-8767-4727-ab0c-900b50a73411",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-80,
180
],
"parameters": {
"color": 5,
"height": 340,
"content": "## AI prompt\nYou are an expert financial analyst tasked with providing an advanced technical analyses of a stock or crypto currency chart provided. Your analysis will be based on various technical indicators and will provide simple insights for novice traders. Just explain to traders were you expect the market is moving. Also warn them this is not a binding advice. Make sure to explain everything in infant language."
},
"typeVersion": 1
}
],
"active": true,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "caf32442-e9c5-466a-8888-9abd2c1b3449",
"connections": {
"OpenAI": {
"main": [
[
{
"node": "Respond to Webhook",
"type": "main",
"index": 0
}
]
]
},
"Webhook": {
"main": [
[
{
"node": "OpenAI",
"type": "main",
"index": 0
}
]
]
}
}
}

131
AI_Research_RAG_and_Data_Analysis/Analyze tradingview.com charts with Chrome extension, N8N and OpenAI.txt
View File

@@ -1,131 +0,0 @@
{
"id": "Q8On8rR6BkmPzDUd",
"meta": {
"instanceId": "f57770b08f6a574802832e927ed1b0063c627ffc5b95965abf0d4a7396150138"
},
"name": "chrome extension backend with AI",
"tags": [],
"nodes": [
{
"id": "0f38fe62-36d9-43da-a992-a3981377e89e",
"name": "Webhook",
"type": "n8n-nodes-base.webhook",
"position": [
-220,
-20
],
"webhookId": "e9a97dd5-f1e7-4d5b-a6f1-be5f0c9eb96c",
"parameters": {
"path": "e9a97dd5-f1e7-4d5b-a6f1-be5f0c9eb96c",
"options": {},
"httpMethod": "POST",
"responseMode": "responseNode"
},
"typeVersion": 2
},
{
"id": "83959562-edf5-4d37-bd11-47186c6a31c7",
"name": "OpenAI",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
-40,
-20
],
"parameters": {
"text": "You are an expert financial analyst tasked with providing an advanced technical analyses of a stock or crypto currency chart provided. Your analysis will be based on various technical indicators and will provide simple insights for novice traders. Just explain to traders were you expect the market is moving. Also warn them this is not a binding advice. Make sure to explain everything in infant language.",
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini",
"cachedResultName": "GPT-4O-MINI"
},
"options": {},
"resource": "image",
"inputType": "base64",
"operation": "analyze"
},
"credentials": {
"openAiApi": {
"id": "8MS1muoK4z86fxUs",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
},
{
"id": "c6f1f833-7ba3-49c5-86df-f586e6bb5975",
"name": "Respond to Webhook",
"type": "n8n-nodes-base.respondToWebhook",
"position": [
140,
-20
],
"parameters": {
"options": {},
"respondWith": "text",
"responseBody": "={{ $json.content }}"
},
"typeVersion": 1.1
},
{
"id": "e3a38a76-283b-4567-a8da-315ef1e2bc4f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-260,
-140
],
"parameters": {
"width": 620,
"height": 300,
"content": "## N8N en OpenAI image analyser"
},
"typeVersion": 1
},
{
"id": "8e7e26db-8767-4727-ab0c-900b50a73411",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-80,
180
],
"parameters": {
"color": 5,
"height": 340,
"content": "## AI prompt\nYou are an expert financial analyst tasked with providing an advanced technical analyses of a stock or crypto currency chart provided. Your analysis will be based on various technical indicators and will provide simple insights for novice traders. Just explain to traders were you expect the market is moving. Also warn them this is not a binding advice. Make sure to explain everything in infant language."
},
"typeVersion": 1
}
],
"active": true,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "caf32442-e9c5-466a-8888-9abd2c1b3449",
"connections": {
"OpenAI": {
"main": [
[
{
"node": "Respond to Webhook",
"type": "main",
"index": 0
}
]
]
},
"Webhook": {
"main": [
[
{
"node": "OpenAI",
"type": "main",
"index": 0
}
]
]
}
}
}

461
AI_Research_RAG_and_Data_Analysis/Automated Hugging Face Paper Summary Fetching & Categorization Workflow.json Normal file
View File

@@ -0,0 +1,461 @@
{
"id": "G8jRDBvwsMkkMiLN",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[3/3] Anomaly detection tool (crops dataset)",
"tags": [
{
"id": "spMntyrlE9ydvWFA",
"name": "anomaly-detection",
"createdAt": "2024-12-08T22:05:15.945Z",
"updatedAt": "2024-12-09T12:50:19.287Z"
}
],
"nodes": [
{
"id": "e01bafec-eb24-44c7-b3c4-a60f91666350",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
180
],
"parameters": {
"color": 6,
"width": 400,
"height": 740,
"content": "We are working here with crops dataset: \nExisting (so not anomalies) crops images in dataset are:\n- 'pearl_millet(bajra)',\n- 'tobacco-plant',\n- 'cherry',\n- 'cotton',\n- 'banana',\n- 'cucumber',\n- 'maize',\n- 'wheat',\n- 'clove',\n- 'jowar',\n- 'olive-tree',\n- 'soyabean',\n- 'coffee-plant',\n- 'rice',\n- 'lemon',\n- 'mustard-oil',\n- 'vigna-radiati(mung)',\n- 'coconut',\n- 'gram',\n- 'pineapple',\n- 'sugarcane',\n- 'sunflower',\n- 'chilli',\n- 'fox_nut(makhana)',\n- 'jute',\n- 'papaya',\n- 'tea',\n- 'cardamom',\n- 'almond'\n"
},
"typeVersion": 1
},
{
"id": "b9943781-de1f-4129-9b81-ed836e9ebb11",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
680,
60
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $('Image URL hardcode').first().json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "47b72bc2-4817-48c6-b517-c1328e402468",
"name": "Get similarity of medoids",
"type": "n8n-nodes-base.httpRequest",
"position": [
940,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.data[0].embedding,\n \"using\": \"voyage\",\n \"limit\": $('Info About Crop Labeled Clusters').first().json.cropsNumber,\n \"with_payload\": true,\n \"filter\": {\n \"must\": [\n { \n \"key\": $('Variables for medoids').first().json.clusterCenterType,\n \"match\": {\n \"value\": true\n }\n }\n ]\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "42d7eb27-ec38-4406-b5c4-27eb45358e93",
"name": "Compare scores",
"type": "n8n-nodes-base.code",
"position": [
1140,
60
],
"parameters": {
"language": "python",
"pythonCode": "points = _input.first()['json']['result']['points']\nthreshold_type = _('Variables for medoids').first()['json']['clusterThresholdCenterType']\n\nmax_score = -1\ncrop_with_max_score = None\nundefined = True\n\nfor center in points:\n if center['score'] >= center['payload'][threshold_type]:\n undefined = False\n if center['score'] > max_score:\n max_score = center['score']\n crop_with_max_score = center['payload']['crop_name']\n\nif undefined:\n result_message = \"ALERT, we might have a new undefined crop!\"\nelse:\n result_message = f\"Looks similar to {crop_with_max_score}\"\n\nreturn [{\n \"json\": {\n \"result\": result_message\n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "23aa604a-ff0b-4948-bcd5-af39300198c0",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
-220
],
"parameters": {
"width": 400,
"height": 380,
"content": "## Crop Anomaly Detection Tool\n### This is the tool that can be used directly for anomalous crops detection. \nIt takes as input (any) **image URL** and returns a **text message** telling if whatever this image depicts is anomalous to the crop dataset stored in Qdrant. \n\n* An Image URL is received via the Execute Workflow Trigger which is used to generate embedding vectors via the Voyage.ai Embeddings API.\n* The returned vectors are used to query the Qdrant collection to determine if the given crop is known by comparing it to **threshold scores** of each image class (crop type).\n* If the image scores lower than all thresholds, then the image is considered an anomaly for the dataset."
},
"typeVersion": 1
},
{
"id": "3a79eca2-44f9-4aee-8a0d-9c7ca2f9149d",
"name": "Variables for medoids",
"type": "n8n-nodes-base.set",
"position": [
-200,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "dbbc1e7b-c63e-4ff1-9524-8ef3e9f6cd48",
"name": "clusterCenterType",
"type": "string",
"value": "is_medoid"
},
{
"id": "a994ce37-2530-4030-acfb-ec777a7ddb05",
"name": "qdrantCloudURL",
"type": "string",
"value": "https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "12f0a9e6-686d-416e-a61b-72d034ec21ba",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "4c88a617-d44f-4776-b457-8a1dffb1d03c",
"name": "clusterThresholdCenterType",
"type": "string",
"value": "is_medoid_cluster_threshold"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "13b25434-bd66-4293-93f1-26c67b9ec7dd",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
260
],
"parameters": {
"color": 6,
"width": 360,
"height": 200,
"content": "**clusterCenterType** - either\n* \"is_text_anchor_medoid\" or\n* \"is_medoid\"\n\n\n**clusterThresholdCenterType** - either\n* \"is_text_anchor_medoid_cluster_threshold\" or\n* \"is_medoid_cluster_threshold\""
},
"typeVersion": 1
},
{
"id": "869b0962-6cae-487d-8230-539a0cc4c14c",
"name": "Info About Crop Labeled Clusters",
"type": "n8n-nodes-base.set",
"position": [
440,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5327b254-b703-4a34-a398-f82edb1d6d6b",
"name": "=cropsNumber",
"type": "number",
"value": "={{ $json.result.hits.length }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "5d3956f8-f43b-439e-b176-a594a21d8011",
"name": "Total Points in Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
40,
60
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/count",
"method": "POST",
"options": {},
"jsonBody": "={\n \"exact\": true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "14ba3db9-3965-4b20-b333-145616d45c3a",
"name": "Each Crop Counts",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/facet",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"key\": \"crop_name\",\n \"limit\": $json.result.count,\n \"exact\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "e37c6758-0556-4a56-ab14-d4df663cb53a",
"name": "Image URL hardcode",
"type": "n8n-nodes-base.set",
"position": [
-480,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "b24ad1a7-0cf8-4acc-9c18-6fe9d58b10f2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-720,
60
],
"parameters": {},
"typeVersion": 1
},
{
"id": "50424f2b-6831-41bf-8de4-81f69d901ce1",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-80
],
"parameters": {
"width": 180,
"height": 120,
"content": "Variables to access Qdrant's collection we uploaded & prepared for anomaly detection in 2 previous pipelines\n"
},
"typeVersion": 1
},
{
"id": "2e8ed3ca-1bba-4214-b34b-376a237842ff",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
-120
],
"parameters": {
"width": 560,
"height": 140,
"content": "These three nodes are needed just to figure out how many different classes (crops) we have in our Qdrant collection: **cropsNumber** (needed in *\"Get similarity of medoids\"* node. \n[Note] *\"Total Points in Collection\"* -> *\"Each Crop Counts\"* were used&explained already in *\"[2/4] Set up medoids (2 types) for anomaly detection (crops dataset)\"* pipeline.\n"
},
"typeVersion": 1
},
{
"id": "e2fa5763-6e97-4ff5-8919-1cb85a3c6968",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
240
],
"parameters": {
"height": 120,
"content": "Here, we're embedding the image passed to this workflow tool with the Voyage embedding model to compare the image to all crop images in the database."
},
"typeVersion": 1
},
{
"id": "cdb6b8d3-f7f4-4d66-850f-ce16c8ed98b9",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
920,
220
],
"parameters": {
"width": 400,
"height": 180,
"content": "Checking how similar the image is to all the centres of clusters (crops).\nIf it's more similar to the thresholds we set up and stored in centres in the previous workflow, the image probably belongs to this crop class; otherwise, it's anomalous to the class. \nIf image is anomalous to all the classes, it's an anomaly."
},
"typeVersion": 1
},
{
"id": "03b4699f-ba43-4f5f-ad69-6f81deea2641",
"name": "Sticky Note22",
"type": "n8n-nodes-base.stickyNote",
"position": [
-620,
580
],
"parameters": {
"color": 4,
"width": 540,
"height": 300,
"content": "### For anomaly detection\n1. The first pipeline is uploading (crops) dataset to Qdrant's collection.\n2. The second pipeline sets up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. **This is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.**\n\n### To recreate it\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/agricultural-crops%2Fcotton%2Fimage%20(36).jpg"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "f67b764b-9e1a-4db0-b9f2-490077a62f74",
"connections": {
"Embed image": {
"main": [
[
{
"node": "Get similarity of medoids",
"type": "main",
"index": 0
}
]
]
},
"Each Crop Counts": {
"main": [
[
{
"node": "Info About Crop Labeled Clusters",
"type": "main",
"index": 0
}
]
]
},
"Image URL hardcode": {
"main": [
[
{
"node": "Variables for medoids",
"type": "main",
"index": 0
}
]
]
},
"Variables for medoids": {
"main": [
[
{
"node": "Total Points in Collection",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image URL hardcode",
"type": "main",
"index": 0
}
]
]
},
"Get similarity of medoids": {
"main": [
[
{
"node": "Compare scores",
"type": "main",
"index": 0
}
]
]
},
"Total Points in Collection": {
"main": [
[
{
"node": "Each Crop Counts",
"type": "main",
"index": 0
}
]
]
},
"Info About Crop Labeled Clusters": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
}
}
}

461
AI_Research_RAG_and_Data_Analysis/Automated Hugging Face Paper Summary Fetching & Categorization Workflow.txt
View File

@@ -1,461 +0,0 @@
{
"id": "G8jRDBvwsMkkMiLN",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[3/3] Anomaly detection tool (crops dataset)",
"tags": [
{
"id": "spMntyrlE9ydvWFA",
"name": "anomaly-detection",
"createdAt": "2024-12-08T22:05:15.945Z",
"updatedAt": "2024-12-09T12:50:19.287Z"
}
],
"nodes": [
{
"id": "e01bafec-eb24-44c7-b3c4-a60f91666350",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
180
],
"parameters": {
"color": 6,
"width": 400,
"height": 740,
"content": "We are working here with crops dataset: \nExisting (so not anomalies) crops images in dataset are:\n- 'pearl_millet(bajra)',\n- 'tobacco-plant',\n- 'cherry',\n- 'cotton',\n- 'banana',\n- 'cucumber',\n- 'maize',\n- 'wheat',\n- 'clove',\n- 'jowar',\n- 'olive-tree',\n- 'soyabean',\n- 'coffee-plant',\n- 'rice',\n- 'lemon',\n- 'mustard-oil',\n- 'vigna-radiati(mung)',\n- 'coconut',\n- 'gram',\n- 'pineapple',\n- 'sugarcane',\n- 'sunflower',\n- 'chilli',\n- 'fox_nut(makhana)',\n- 'jute',\n- 'papaya',\n- 'tea',\n- 'cardamom',\n- 'almond'\n"
},
"typeVersion": 1
},
{
"id": "b9943781-de1f-4129-9b81-ed836e9ebb11",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
680,
60
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $('Image URL hardcode').first().json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "47b72bc2-4817-48c6-b517-c1328e402468",
"name": "Get similarity of medoids",
"type": "n8n-nodes-base.httpRequest",
"position": [
940,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.data[0].embedding,\n \"using\": \"voyage\",\n \"limit\": $('Info About Crop Labeled Clusters').first().json.cropsNumber,\n \"with_payload\": true,\n \"filter\": {\n \"must\": [\n { \n \"key\": $('Variables for medoids').first().json.clusterCenterType,\n \"match\": {\n \"value\": true\n }\n }\n ]\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "42d7eb27-ec38-4406-b5c4-27eb45358e93",
"name": "Compare scores",
"type": "n8n-nodes-base.code",
"position": [
1140,
60
],
"parameters": {
"language": "python",
"pythonCode": "points = _input.first()['json']['result']['points']\nthreshold_type = _('Variables for medoids').first()['json']['clusterThresholdCenterType']\n\nmax_score = -1\ncrop_with_max_score = None\nundefined = True\n\nfor center in points:\n if center['score'] >= center['payload'][threshold_type]:\n undefined = False\n if center['score'] > max_score:\n max_score = center['score']\n crop_with_max_score = center['payload']['crop_name']\n\nif undefined:\n result_message = \"ALERT, we might have a new undefined crop!\"\nelse:\n result_message = f\"Looks similar to {crop_with_max_score}\"\n\nreturn [{\n \"json\": {\n \"result\": result_message\n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "23aa604a-ff0b-4948-bcd5-af39300198c0",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
-220
],
"parameters": {
"width": 400,
"height": 380,
"content": "## Crop Anomaly Detection Tool\n### This is the tool that can be used directly for anomalous crops detection. \nIt takes as input (any) **image URL** and returns a **text message** telling if whatever this image depicts is anomalous to the crop dataset stored in Qdrant. \n\n* An Image URL is received via the Execute Workflow Trigger which is used to generate embedding vectors via the Voyage.ai Embeddings API.\n* The returned vectors are used to query the Qdrant collection to determine if the given crop is known by comparing it to **threshold scores** of each image class (crop type).\n* If the image scores lower than all thresholds, then the image is considered an anomaly for the dataset."
},
"typeVersion": 1
},
{
"id": "3a79eca2-44f9-4aee-8a0d-9c7ca2f9149d",
"name": "Variables for medoids",
"type": "n8n-nodes-base.set",
"position": [
-200,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "dbbc1e7b-c63e-4ff1-9524-8ef3e9f6cd48",
"name": "clusterCenterType",
"type": "string",
"value": "is_medoid"
},
{
"id": "a994ce37-2530-4030-acfb-ec777a7ddb05",
"name": "qdrantCloudURL",
"type": "string",
"value": "https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "12f0a9e6-686d-416e-a61b-72d034ec21ba",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "4c88a617-d44f-4776-b457-8a1dffb1d03c",
"name": "clusterThresholdCenterType",
"type": "string",
"value": "is_medoid_cluster_threshold"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "13b25434-bd66-4293-93f1-26c67b9ec7dd",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
260
],
"parameters": {
"color": 6,
"width": 360,
"height": 200,
"content": "**clusterCenterType** - either\n* \"is_text_anchor_medoid\" or\n* \"is_medoid\"\n\n\n**clusterThresholdCenterType** - either\n* \"is_text_anchor_medoid_cluster_threshold\" or\n* \"is_medoid_cluster_threshold\""
},
"typeVersion": 1
},
{
"id": "869b0962-6cae-487d-8230-539a0cc4c14c",
"name": "Info About Crop Labeled Clusters",
"type": "n8n-nodes-base.set",
"position": [
440,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5327b254-b703-4a34-a398-f82edb1d6d6b",
"name": "=cropsNumber",
"type": "number",
"value": "={{ $json.result.hits.length }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "5d3956f8-f43b-439e-b176-a594a21d8011",
"name": "Total Points in Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
40,
60
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/count",
"method": "POST",
"options": {},
"jsonBody": "={\n \"exact\": true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "14ba3db9-3965-4b20-b333-145616d45c3a",
"name": "Each Crop Counts",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/facet",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"key\": \"crop_name\",\n \"limit\": $json.result.count,\n \"exact\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "e37c6758-0556-4a56-ab14-d4df663cb53a",
"name": "Image URL hardcode",
"type": "n8n-nodes-base.set",
"position": [
-480,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "b24ad1a7-0cf8-4acc-9c18-6fe9d58b10f2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-720,
60
],
"parameters": {},
"typeVersion": 1
},
{
"id": "50424f2b-6831-41bf-8de4-81f69d901ce1",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-80
],
"parameters": {
"width": 180,
"height": 120,
"content": "Variables to access Qdrant's collection we uploaded & prepared for anomaly detection in 2 previous pipelines\n"
},
"typeVersion": 1
},
{
"id": "2e8ed3ca-1bba-4214-b34b-376a237842ff",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
-120
],
"parameters": {
"width": 560,
"height": 140,
"content": "These three nodes are needed just to figure out how many different classes (crops) we have in our Qdrant collection: **cropsNumber** (needed in *\"Get similarity of medoids\"* node. \n[Note] *\"Total Points in Collection\"* -> *\"Each Crop Counts\"* were used&explained already in *\"[2/4] Set up medoids (2 types) for anomaly detection (crops dataset)\"* pipeline.\n"
},
"typeVersion": 1
},
{
"id": "e2fa5763-6e97-4ff5-8919-1cb85a3c6968",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
240
],
"parameters": {
"height": 120,
"content": "Here, we're embedding the image passed to this workflow tool with the Voyage embedding model to compare the image to all crop images in the database."
},
"typeVersion": 1
},
{
"id": "cdb6b8d3-f7f4-4d66-850f-ce16c8ed98b9",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
920,
220
],
"parameters": {
"width": 400,
"height": 180,
"content": "Checking how similar the image is to all the centres of clusters (crops).\nIf it's more similar to the thresholds we set up and stored in centres in the previous workflow, the image probably belongs to this crop class; otherwise, it's anomalous to the class. \nIf image is anomalous to all the classes, it's an anomaly."
},
"typeVersion": 1
},
{
"id": "03b4699f-ba43-4f5f-ad69-6f81deea2641",
"name": "Sticky Note22",
"type": "n8n-nodes-base.stickyNote",
"position": [
-620,
580
],
"parameters": {
"color": 4,
"width": 540,
"height": 300,
"content": "### For anomaly detection\n1. The first pipeline is uploading (crops) dataset to Qdrant's collection.\n2. The second pipeline sets up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. **This is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.**\n\n### To recreate it\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/agricultural-crops%2Fcotton%2Fimage%20(36).jpg"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "f67b764b-9e1a-4db0-b9f2-490077a62f74",
"connections": {
"Embed image": {
"main": [
[
{
"node": "Get similarity of medoids",
"type": "main",
"index": 0
}
]
]
},
"Each Crop Counts": {
"main": [
[
{
"node": "Info About Crop Labeled Clusters",
"type": "main",
"index": 0
}
]
]
},
"Image URL hardcode": {
"main": [
[
{
"node": "Variables for medoids",
"type": "main",
"index": 0
}
]
]
},
"Variables for medoids": {
"main": [
[
{
"node": "Total Points in Collection",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image URL hardcode",
"type": "main",
"index": 0
}
]
]
},
"Get similarity of medoids": {
"main": [
[
{
"node": "Compare scores",
"type": "main",
"index": 0
}
]
]
},
"Total Points in Collection": {
"main": [
[
{
"node": "Each Crop Counts",
"type": "main",
"index": 0
}
]
]
},
"Info About Crop Labeled Clusters": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
}
}
}

1023
AI_Research_RAG_and_Data_Analysis/Autonomous AI crawler.json Normal file
View File

File diff suppressed because one or more lines are too long

1023
AI_Research_RAG_and_Data_Analysis/Autonomous AI crawler.txt
View File

File diff suppressed because one or more lines are too long

477
AI_Research_RAG_and_Data_Analysis/Build Your Own Image Search Using AI Object Detection, CDN and ElasticSearchBuild Your Own Image Search Using AI Object Detection, CDN and ElasticSearch.json Normal file
View File

@@ -0,0 +1,477 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "6359f725-1ede-4b05-bc19-05a7e85c0865",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
680,
292
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9e1e61c7-f5fd-4e8a-99a6-ccc5a24f5528",
"name": "Fetch Source Image",
"type": "n8n-nodes-base.httpRequest",
"position": [
1000,
292
],
"parameters": {
"url": "={{ $json.source_image }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "9b1b94cf-3a7d-4c43-ab6c-8df9824b5667",
"name": "Split Out Results Only",
"type": "n8n-nodes-base.splitOut",
"position": [
1428,
323
],
"parameters": {
"options": {},
"fieldToSplitOut": "result"
},
"typeVersion": 1
},
{
"id": "fcbaf6c3-2aee-4ea1-9c5e-2833dd7a9f50",
"name": "Filter Score >= 0.9",
"type": "n8n-nodes-base.filter",
"position": [
1608,
323
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "367d83ef-8ecf-41fe-858c-9bfd78b0ae9f",
"operator": {
"type": "number",
"operation": "gte"
},
"leftValue": "={{ $json.score }}",
"rightValue": 0.9
}
]
}
},
"typeVersion": 2
},
{
"id": "954ce7b0-ef82-4203-8706-17cfa5e5e3ff",
"name": "Crop Object From Image",
"type": "n8n-nodes-base.editImage",
"position": [
2080,
432
],
"parameters": {
"width": "={{ $json.box.xmax - $json.box.xmin }}",
"height": "={{ $json.box.ymax - $json.box.ymin }}",
"options": {
"format": "jpeg",
"fileName": "={{ $binary.data.fileName.split('.')[0].urlEncode()+'-'+$json.label.urlEncode() + '-' + $itemIndex }}.jpg"
},
"operation": "crop",
"positionX": "={{ $json.box.xmin }}",
"positionY": "={{ $json.box.ymin }}"
},
"typeVersion": 1
},
{
"id": "40027456-4bf9-4eea-8d71-aa28e69b29e5",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
840,
292
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9e95d951-8530-4a80-bd00-6bb55623a71f",
"name": "CLOUDFLARE_ACCOUNT_ID",
"type": "string",
"value": ""
},
{
"id": "66807a90-63a1-4d4e-886e-e8abf3019a34",
"name": "model",
"type": "string",
"value": "@cf/facebook/detr-resnet-50"
},
{
"id": "a13ccde6-e6e3-46f4-afa3-2134af7bc765",
"name": "source_image",
"type": "string",
"value": "https://images.pexels.com/photos/2293367/pexels-photo-2293367.jpeg?auto=compress&cs=tinysrgb&w=600"
},
{
"id": "0734fc55-b414-47f7-8b3e-5c880243f3ed",
"name": "elasticsearch_index",
"type": "string",
"value": "n8n-image-search"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "c3d8c5e3-546e-472c-9e6e-091cf5cee3c3",
"name": "Use Detr-Resnet-50 Object Classification",
"type": "n8n-nodes-base.httpRequest",
"position": [
1248,
324
],
"parameters": {
"url": "=https://api.cloudflare.com/client/v4/accounts/{{ $('Set Variables').item.json.CLOUDFLARE_ACCOUNT_ID }}/ai/run/{{ $('Set Variables').item.json.model }}",
"method": "POST",
"options": {},
"sendBody": true,
"contentType": "binaryData",
"authentication": "predefinedCredentialType",
"inputDataFieldName": "data",
"nodeCredentialType": "cloudflareApi"
},
"credentials": {
"cloudflareApi": {
"id": "qOynkQdBH48ofOSS",
"name": "Cloudflare account"
}
},
"typeVersion": 4.2
},
{
"id": "3c7aa2fc-9ca1-41ba-a10d-aa5930d45f18",
"name": "Upload to Cloudinary",
"type": "n8n-nodes-base.httpRequest",
"position": [
2380,
380
],
"parameters": {
"url": "https://api.cloudinary.com/v1_1/daglih2g8/image/upload",
"method": "POST",
"options": {},
"sendBody": true,
"sendQuery": true,
"contentType": "multipart-form-data",
"authentication": "genericCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "file",
"parameterType": "formBinaryData",
"inputDataFieldName": "data"
}
]
},
"genericAuthType": "httpQueryAuth",
"queryParameters": {
"parameters": [
{
"name": "upload_preset",
"value": "n8n-workflows-preset"
}
]
}
},
"credentials": {
"httpQueryAuth": {
"id": "sT9jeKzZiLJ3bVPz",
"name": "Cloudinary API"
}
},
"typeVersion": 4.2
},
{
"id": "3c4e1f04-a0ba-4cce-b82a-aa3eadc4e7e1",
"name": "Create Docs In Elasticsearch",
"type": "n8n-nodes-base.elasticsearch",
"position": [
2580,
380
],
"parameters": {
"indexId": "={{ $('Set Variables').item.json.elasticsearch_index }}",
"options": {},
"fieldsUi": {
"fieldValues": [
{
"fieldId": "image_url",
"fieldValue": "={{ $json.secure_url.replace('upload','upload/f_auto,q_auto') }}"
},
{
"fieldId": "source_image_url",
"fieldValue": "={{ $('Set Variables').item.json.source_image }}"
},
{
"fieldId": "label",
"fieldValue": "={{ $('Crop Object From Image').item.json.label }}"
},
{
"fieldId": "metadata",
"fieldValue": "={{ JSON.stringify(Object.assign($('Crop Object From Image').item.json, { filename: $json.original_filename })) }}"
}
]
},
"operation": "create",
"additionalFields": {}
},
"credentials": {
"elasticsearchApi": {
"id": "dRuuhAgS7AF0mw0S",
"name": "Elasticsearch account"
}
},
"typeVersion": 1
},
{
"id": "292c9821-c123-44fa-9ba1-c37bf84079bc",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
120
],
"parameters": {
"color": 7,
"width": 541.1455500767354,
"height": 381.6388867600897,
"content": "## 1. Get Source Image\n[Read more about setting variables for your workflow](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.set)\n\nFor this demo, we'll manually define an image to process. In production however, this image can come from a variety of sources such as drives, webhooks and more."
},
"typeVersion": 1
},
{
"id": "863271dc-fb9d-4211-972d-6b57336073b4",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1180,
80
],
"parameters": {
"color": 7,
"width": 579.7748008857744,
"height": 437.4680103498263,
"content": "## 2. Use Detr-Resnet-50 Object Classification\n[Learn more about Cloudflare Workers AI](https://developers.cloudflare.com/workers-ai/)\n\nNot all AI workflows need an LLM! As in this example, we're using a non-LLM vision model to parse the source image and return what objects are contained within. The image search feature we're building will be based on the objects in the image making for a much more granular search via object association.\n\nWe'll use the Cloudflare Workers AI service which conveniently provides this model via API use."
},
"typeVersion": 1
},
{
"id": "b73b45da-0436-4099-b538-c6b3b84822f2",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1800,
260
],
"parameters": {
"color": 7,
"width": 466.35460775498495,
"height": 371.9272151757119,
"content": "## 3. Crop Objects Out of Source Image\n[Read more about Editing Images in n8n](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.editimage)\n\nWith our objects identified by their bounding boxes, we can \"cut\" them out of the source image as separate images."
},
"typeVersion": 1
},
{
"id": "465bd842-8a35-49d8-a9ff-c30d164620db",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
2300,
180
],
"parameters": {
"color": 7,
"width": 478.20345439832454,
"height": 386.06196032653685,
"content": "## 4. Index Object Images In ElasticSearch\n[Read more about using ElasticSearch](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.elasticsearch)\n\nBy storing the newly created object images externally and indexing them in Elasticsearch, we now have a foundation for our Image Search service which queries by object association."
},
"typeVersion": 1
},
{
"id": "6a04b4b5-7830-410d-9b5b-79acb0b1c78b",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
1800,
-220
],
"parameters": {
"color": 7,
"width": 328.419768654291,
"height": 462.65463700396174,
"content": "Fig 1. Result of Classification\n![image of classification](https://res.cloudinary.com/daglih2g8/image/upload/f_auto,q_auto,w_300/v1/n8n-workflows/ywtzjcmqrypihci1npgh)"
},
"typeVersion": 1
},
{
"id": "8f607951-ba41-4362-8323-e8b4b96ad122",
"name": "Fetch Source Image Again",
"type": "n8n-nodes-base.httpRequest",
"position": [
1880,
432
],
"parameters": {
"url": "={{ $('Set Variables').item.json.source_image }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "6933f67d-276b-4908-8602-654aa352a68b",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
120
],
"parameters": {
"width": 359.6648027457353,
"height": 352.41026669883723,
"content": "## Try It Out!\n### This workflow does the following:\n* Downloads an image\n* Uses an object classification AI model to identify objects in the image.\n* Crops the objects out from the original image into new image files.\n* Indexes the image's object in an Elasticsearch Database to enable image search.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "35615ed5-43e8-43f0-95fe-1f95a1177d69",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
800,
280
],
"parameters": {
"width": 172.9365918827757,
"height": 291.6881468483679,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set your variables here first!"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Set Variables": {
"main": [
[
{
"node": "Fetch Source Image",
"type": "main",
"index": 0
}
]
]
},
"Fetch Source Image": {
"main": [
[
{
"node": "Use Detr-Resnet-50 Object Classification",
"type": "main",
"index": 0
}
]
]
},
"Filter Score >= 0.9": {
"main": [
[
{
"node": "Fetch Source Image Again",
"type": "main",
"index": 0
}
]
]
},
"Upload to Cloudinary": {
"main": [
[
{
"node": "Create Docs In Elasticsearch",
"type": "main",
"index": 0
}
]
]
},
"Crop Object From Image": {
"main": [
[
{
"node": "Upload to Cloudinary",
"type": "main",
"index": 0
}
]
]
},
"Split Out Results Only": {
"main": [
[
{
"node": "Filter Score >= 0.9",
"type": "main",
"index": 0
}
]
]
},
"Fetch Source Image Again": {
"main": [
[
{
"node": "Crop Object From Image",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Use Detr-Resnet-50 Object Classification": {
"main": [
[
{
"node": "Split Out Results Only",
"type": "main",
"index": 0
}
]
]
}
}
}

477
AI_Research_RAG_and_Data_Analysis/Build Your Own Image Search Using AI Object Detection, CDN and ElasticSearchBuild Your Own Image Search Using AI Object Detection, CDN and ElasticSearch.txt
View File

@@ -1,477 +0,0 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "6359f725-1ede-4b05-bc19-05a7e85c0865",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
680,
292
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9e1e61c7-f5fd-4e8a-99a6-ccc5a24f5528",
"name": "Fetch Source Image",
"type": "n8n-nodes-base.httpRequest",
"position": [
1000,
292
],
"parameters": {
"url": "={{ $json.source_image }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "9b1b94cf-3a7d-4c43-ab6c-8df9824b5667",
"name": "Split Out Results Only",
"type": "n8n-nodes-base.splitOut",
"position": [
1428,
323
],
"parameters": {
"options": {},
"fieldToSplitOut": "result"
},
"typeVersion": 1
},
{
"id": "fcbaf6c3-2aee-4ea1-9c5e-2833dd7a9f50",
"name": "Filter Score >= 0.9",
"type": "n8n-nodes-base.filter",
"position": [
1608,
323
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "367d83ef-8ecf-41fe-858c-9bfd78b0ae9f",
"operator": {
"type": "number",
"operation": "gte"
},
"leftValue": "={{ $json.score }}",
"rightValue": 0.9
}
]
}
},
"typeVersion": 2
},
{
"id": "954ce7b0-ef82-4203-8706-17cfa5e5e3ff",
"name": "Crop Object From Image",
"type": "n8n-nodes-base.editImage",
"position": [
2080,
432
],
"parameters": {
"width": "={{ $json.box.xmax - $json.box.xmin }}",
"height": "={{ $json.box.ymax - $json.box.ymin }}",
"options": {
"format": "jpeg",
"fileName": "={{ $binary.data.fileName.split('.')[0].urlEncode()+'-'+$json.label.urlEncode() + '-' + $itemIndex }}.jpg"
},
"operation": "crop",
"positionX": "={{ $json.box.xmin }}",
"positionY": "={{ $json.box.ymin }}"
},
"typeVersion": 1
},
{
"id": "40027456-4bf9-4eea-8d71-aa28e69b29e5",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
840,
292
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9e95d951-8530-4a80-bd00-6bb55623a71f",
"name": "CLOUDFLARE_ACCOUNT_ID",
"type": "string",
"value": ""
},
{
"id": "66807a90-63a1-4d4e-886e-e8abf3019a34",
"name": "model",
"type": "string",
"value": "@cf/facebook/detr-resnet-50"
},
{
"id": "a13ccde6-e6e3-46f4-afa3-2134af7bc765",
"name": "source_image",
"type": "string",
"value": "https://images.pexels.com/photos/2293367/pexels-photo-2293367.jpeg?auto=compress&cs=tinysrgb&w=600"
},
{
"id": "0734fc55-b414-47f7-8b3e-5c880243f3ed",
"name": "elasticsearch_index",
"type": "string",
"value": "n8n-image-search"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "c3d8c5e3-546e-472c-9e6e-091cf5cee3c3",
"name": "Use Detr-Resnet-50 Object Classification",
"type": "n8n-nodes-base.httpRequest",
"position": [
1248,
324
],
"parameters": {
"url": "=https://api.cloudflare.com/client/v4/accounts/{{ $('Set Variables').item.json.CLOUDFLARE_ACCOUNT_ID }}/ai/run/{{ $('Set Variables').item.json.model }}",
"method": "POST",
"options": {},
"sendBody": true,
"contentType": "binaryData",
"authentication": "predefinedCredentialType",
"inputDataFieldName": "data",
"nodeCredentialType": "cloudflareApi"
},
"credentials": {
"cloudflareApi": {
"id": "qOynkQdBH48ofOSS",
"name": "Cloudflare account"
}
},
"typeVersion": 4.2
},
{
"id": "3c7aa2fc-9ca1-41ba-a10d-aa5930d45f18",
"name": "Upload to Cloudinary",
"type": "n8n-nodes-base.httpRequest",
"position": [
2380,
380
],
"parameters": {
"url": "https://api.cloudinary.com/v1_1/daglih2g8/image/upload",
"method": "POST",
"options": {},
"sendBody": true,
"sendQuery": true,
"contentType": "multipart-form-data",
"authentication": "genericCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "file",
"parameterType": "formBinaryData",
"inputDataFieldName": "data"
}
]
},
"genericAuthType": "httpQueryAuth",
"queryParameters": {
"parameters": [
{
"name": "upload_preset",
"value": "n8n-workflows-preset"
}
]
}
},
"credentials": {
"httpQueryAuth": {
"id": "sT9jeKzZiLJ3bVPz",
"name": "Cloudinary API"
}
},
"typeVersion": 4.2
},
{
"id": "3c4e1f04-a0ba-4cce-b82a-aa3eadc4e7e1",
"name": "Create Docs In Elasticsearch",
"type": "n8n-nodes-base.elasticsearch",
"position": [
2580,
380
],
"parameters": {
"indexId": "={{ $('Set Variables').item.json.elasticsearch_index }}",
"options": {},
"fieldsUi": {
"fieldValues": [
{
"fieldId": "image_url",
"fieldValue": "={{ $json.secure_url.replace('upload','upload/f_auto,q_auto') }}"
},
{
"fieldId": "source_image_url",
"fieldValue": "={{ $('Set Variables').item.json.source_image }}"
},
{
"fieldId": "label",
"fieldValue": "={{ $('Crop Object From Image').item.json.label }}"
},
{
"fieldId": "metadata",
"fieldValue": "={{ JSON.stringify(Object.assign($('Crop Object From Image').item.json, { filename: $json.original_filename })) }}"
}
]
},
"operation": "create",
"additionalFields": {}
},
"credentials": {
"elasticsearchApi": {
"id": "dRuuhAgS7AF0mw0S",
"name": "Elasticsearch account"
}
},
"typeVersion": 1
},
{
"id": "292c9821-c123-44fa-9ba1-c37bf84079bc",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
120
],
"parameters": {
"color": 7,
"width": 541.1455500767354,
"height": 381.6388867600897,
"content": "## 1. Get Source Image\n[Read more about setting variables for your workflow](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.set)\n\nFor this demo, we'll manually define an image to process. In production however, this image can come from a variety of sources such as drives, webhooks and more."
},
"typeVersion": 1
},
{
"id": "863271dc-fb9d-4211-972d-6b57336073b4",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1180,
80
],
"parameters": {
"color": 7,
"width": 579.7748008857744,
"height": 437.4680103498263,
"content": "## 2. Use Detr-Resnet-50 Object Classification\n[Learn more about Cloudflare Workers AI](https://developers.cloudflare.com/workers-ai/)\n\nNot all AI workflows need an LLM! As in this example, we're using a non-LLM vision model to parse the source image and return what objects are contained within. The image search feature we're building will be based on the objects in the image making for a much more granular search via object association.\n\nWe'll use the Cloudflare Workers AI service which conveniently provides this model via API use."
},
"typeVersion": 1
},
{
"id": "b73b45da-0436-4099-b538-c6b3b84822f2",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1800,
260
],
"parameters": {
"color": 7,
"width": 466.35460775498495,
"height": 371.9272151757119,
"content": "## 3. Crop Objects Out of Source Image\n[Read more about Editing Images in n8n](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.editimage)\n\nWith our objects identified by their bounding boxes, we can \"cut\" them out of the source image as separate images."
},
"typeVersion": 1
},
{
"id": "465bd842-8a35-49d8-a9ff-c30d164620db",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
2300,
180
],
"parameters": {
"color": 7,
"width": 478.20345439832454,
"height": 386.06196032653685,
"content": "## 4. Index Object Images In ElasticSearch\n[Read more about using ElasticSearch](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.elasticsearch)\n\nBy storing the newly created object images externally and indexing them in Elasticsearch, we now have a foundation for our Image Search service which queries by object association."
},
"typeVersion": 1
},
{
"id": "6a04b4b5-7830-410d-9b5b-79acb0b1c78b",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
1800,
-220
],
"parameters": {
"color": 7,
"width": 328.419768654291,
"height": 462.65463700396174,
"content": "Fig 1. Result of Classification\n![image of classification](https://res.cloudinary.com/daglih2g8/image/upload/f_auto,q_auto,w_300/v1/n8n-workflows/ywtzjcmqrypihci1npgh)"
},
"typeVersion": 1
},
{
"id": "8f607951-ba41-4362-8323-e8b4b96ad122",
"name": "Fetch Source Image Again",
"type": "n8n-nodes-base.httpRequest",
"position": [
1880,
432
],
"parameters": {
"url": "={{ $('Set Variables').item.json.source_image }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "6933f67d-276b-4908-8602-654aa352a68b",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
120
],
"parameters": {
"width": 359.6648027457353,
"height": 352.41026669883723,
"content": "## Try It Out!\n### This workflow does the following:\n* Downloads an image\n* Uses an object classification AI model to identify objects in the image.\n* Crops the objects out from the original image into new image files.\n* Indexes the image's object in an Elasticsearch Database to enable image search.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "35615ed5-43e8-43f0-95fe-1f95a1177d69",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
800,
280
],
"parameters": {
"width": 172.9365918827757,
"height": 291.6881468483679,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set your variables here first!"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Set Variables": {
"main": [
[
{
"node": "Fetch Source Image",
"type": "main",
"index": 0
}
]
]
},
"Fetch Source Image": {
"main": [
[
{
"node": "Use Detr-Resnet-50 Object Classification",
"type": "main",
"index": 0
}
]
]
},
"Filter Score >= 0.9": {
"main": [
[
{
"node": "Fetch Source Image Again",
"type": "main",
"index": 0
}
]
]
},
"Upload to Cloudinary": {
"main": [
[
{
"node": "Create Docs In Elasticsearch",
"type": "main",
"index": 0
}
]
]
},
"Crop Object From Image": {
"main": [
[
{
"node": "Upload to Cloudinary",
"type": "main",
"index": 0
}
]
]
},
"Split Out Results Only": {
"main": [
[
{
"node": "Filter Score >= 0.9",
"type": "main",
"index": 0
}
]
]
},
"Fetch Source Image Again": {
"main": [
[
{
"node": "Crop Object From Image",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Use Detr-Resnet-50 Object Classification": {
"main": [
[
{
"node": "Split Out Results Only",
"type": "main",
"index": 0
}
]
]
}
}
}

940
AI_Research_RAG_and_Data_Analysis/Build a Financial Documents Assistant using Qdrant and Mistral.ai.json Normal file
View File

@@ -0,0 +1,940 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "c5525f47-4d91-4b98-87bb-566b90da64a1",
"name": "Local File Trigger",
"type": "n8n-nodes-base.localFileTrigger",
"position": [
660,
700
],
"parameters": {
"path": "/home/node/host_mount/local_file_search",
"events": [
"add",
"change",
"unlink"
],
"options": {
"awaitWriteFinish": true
},
"triggerOn": "folder"
},
"typeVersion": 1
},
{
"id": "804334d6-e34d-40d1-9555-b331ffe66f6f",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
664.5766613599001,
881.8474780113352
],
"parameters": {},
"typeVersion": 1
},
{
"id": "7ab0e284-b667-4d1f-8ceb-fb05e4081a06",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
840,
700
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "35ea70c4-8669-4975-a68d-bbaa094713c0",
"name": "directory",
"type": "string",
"value": "/home/node/BankStatements"
},
{
"id": "1d081d19-ff4e-462a-9cbe-7af2244bf87f",
"name": "file_added",
"type": "string",
"value": "={{ $json.event === 'add' && $json.path || ''}}"
},
{
"id": "18f8dc03-51ca-48c7-947f-87ce8e1979bf",
"name": "file_changed",
"type": "string",
"value": "={{ $json.event === 'change' && $json.path || '' }}"
},
{
"id": "65074ff7-037b-4b3b-b2c3-8a61755ab43b",
"name": "file_deleted",
"type": "string",
"value": "={{ $json.event === 'unlink' && $json.path || '' }}"
},
{
"id": "9a1902e7-f94d-4d1f-9006-91c67354d3e8",
"name": "qdrant_collection",
"type": "string",
"value": "local_file_search"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "76173972-ceca-43a4-b85f-00b41f774304",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
580,
460
],
"parameters": {
"color": 7,
"width": 665.0909497859384,
"height": 596.8351502261468,
"content": "## Step 1. Select the target folder\n[Read more about local file trigger](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.localfiletrigger)\n\nIn this workflow, we'll monitor a specific folder on disk that n8n has access to. Since we're using docker, we can either use the n8n volume or mount a folder from the host machine.\n\nThe local file trigger is useful to execute the workflow whenever changes are made to our target folder."
},
"typeVersion": 1
},
{
"id": "eda839f7-dde4-4d1f-9fe6-692df4ac7282",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
184.57666135990007,
461.84747801133517
],
"parameters": {
"width": 372.51107341403605,
"height": 356.540665091993,
"content": "## Try It Out!\n### This workflow does the following:\n* Monitors a target folder for changes using the local file trigger\n* Synchronises files in the target folder with their vectors in Qdrant\n* Mistral AI is used to create a Q&A AI agent on all files in the target folder\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "f82f6de0-af8f-4fdf-a733-f59ba4fed02f",
"name": "Read File",
"type": "n8n-nodes-base.readWriteFile",
"position": [
1340,
1120
],
"parameters": {
"options": {},
"fileSelector": "={{ $json.file_added }}"
},
"typeVersion": 1
},
{
"id": "7354a080-051b-479f-97b1-49cc0c14c9d8",
"name": "Embeddings Mistral Cloud",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
1720,
1280
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "a1ad45ff-a882-4aed-82e2-cad2483cf4e8",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
1820,
1280
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "filter_by_filename",
"value": "={{ $json.file_location }}"
},
{
"name": "filter_by_created_month",
"value": "={{ $now.year + '-' + $now.monthShort }}"
},
{
"name": "filter_by_created_week",
"value": "={{ $now.year + '-' + $now.monthShort + '-W' + $now.weekNumber }}"
}
]
}
},
"jsonData": "={{ $json.data }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "0b0e29b9-8873-4074-94dc-9f0364c28835",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
1840,
1400
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "c0555ba6-a1bd-4aa9-a340-a9c617f8e6db",
"name": "Prepare Embedding Document",
"type": "n8n-nodes-base.set",
"position": [
1520,
1120
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "41a1d4ca-e5a5-4fb9-b249-8796ae759b33",
"name": "data",
"type": "string",
"value": "=## file location\n{{ [$json.directory, $json.fileName].join('/') }}\n## file created\n{{ $now.toISO() }}\n## file contents\n{{ $input.item.binary.data.data.base64Decode() }}"
},
{
"id": "c091704d-b81c-448b-8c90-156ef568b871",
"name": "file_location",
"type": "string",
"value": "={{ [$json.directory, $json.fileName].join('/') }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "ffe8c363-0809-4d21-aa8f-34b0fc2dc57f",
"name": "Chat Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
2280,
680
],
"webhookId": "37587fe0-b8db-4012-90a7-1f65b9bfd0df",
"parameters": {},
"typeVersion": 1
},
{
"id": "8d958669-60be-4bb2-80fc-2a6c7c7bfae6",
"name": "Question and Answer Chain",
"type": "@n8n/n8n-nodes-langchain.chainRetrievalQa",
"position": [
2500,
680
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "f143e438-8176-4923-a866-3f9a2a16793d",
"name": "Mistral Cloud Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatMistralCloud",
"position": [
2500,
840
],
"parameters": {
"model": "mistral-small-2402",
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "06dd8f4c-3b66-43e0-85c8-ec222e275f87",
"name": "Vector Store Retriever",
"type": "@n8n/n8n-nodes-langchain.retrieverVectorStore",
"position": [
2620,
840
],
"parameters": {},
"typeVersion": 1
},
{
"id": "2fdabcb5-a7a7-4e02-8c1b-9190e2e52385",
"name": "Embeddings Mistral Cloud1",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
2620,
1080
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "e5664534-de07-481f-87dd-68d7d0715baa",
"name": "Remap for File_Added Flow",
"type": "n8n-nodes-base.set",
"position": [
1920,
700
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "840219e1-ed47-4b00-83fd-6b3c0bd71650",
"name": "file_added",
"type": "string",
"value": "={{ $('Set Variables').item.json.file_changed }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "1fd14832-aafe-4d72-b4f2-7afc72df97dc",
"name": "Search For Existing Point",
"type": "n8n-nodes-base.httpRequest",
"position": [
1340,
280
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/scroll",
"method": "POST",
"options": {},
"jsonBody": "={\n \"filter\": {\n \"must\": [\n {\n \"key\": \"metadata.filter_by_filename\",\n \"match\": {\n \"value\": \"{{ $json.file_changed }}\"\n }\n }\n ]\n },\n \"limit\": 1,\n \"with_payload\": false,\n \"with_vector\": false\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "b5fa817f-82d6-41dd-9817-4c1dd9137b76",
"name": "Has Existing Point?",
"type": "n8n-nodes-base.if",
"position": [
1520,
280
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0392bac0-8fb5-406b-b59f-575edf5ab30d",
"operator": {
"type": "array",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.result.points }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "b0fa4fa4-5d1b-4a12-b8ba-a10d71f31f94",
"name": "Delete Existing Point",
"type": "n8n-nodes-base.httpRequest",
"position": [
1720,
700
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/delete",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "points",
"value": "={{ $json.result.points.map(point => point.id) }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "5408adfe-4d6b-407c-aac7-e87c9b1a1592",
"name": "Search For Existing Point1",
"type": "n8n-nodes-base.httpRequest",
"position": [
1340,
700
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/scroll",
"method": "POST",
"options": {},
"jsonBody": "={\n \"filter\": {\n \"must\": [\n {\n \"key\": \"metadata.filter_by_filename\",\n \"match\": {\n \"value\": \"{{ $json.file_changed }}\"\n }\n }\n ]\n },\n \"limit\": 1,\n \"with_payload\": false,\n \"with_vector\": false\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "fac43587-0d24-4d6e-a0d5-8cc8f9615967",
"name": "Has Existing Point?1",
"type": "n8n-nodes-base.if",
"position": [
1520,
700
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0392bac0-8fb5-406b-b59f-575edf5ab30d",
"operator": {
"type": "array",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.result.points }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "010baacd-fac1-4cc1-86bf-9d6ef11916fe",
"name": "Delete Existing Point1",
"type": "n8n-nodes-base.httpRequest",
"position": [
1700,
280
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/delete",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "points",
"value": "={{ $json.result.points.map(point => point.id) }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "2d6fb29c-2fac-41de-9ad0-cc781b246378",
"name": "Handle File Event",
"type": "n8n-nodes-base.switch",
"position": [
1000,
700
],
"parameters": {
"rules": {
"values": [
{
"outputKey": "file_deleted",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "a1f6d86a-9805-4d0e-ac70-90c9cf0ad339",
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_deleted }}",
"rightValue": ""
}
]
},
"renameOutput": true
},
{
"outputKey": "file_changed",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "d15cde67-b5b0-4676-b4fb-ead749147392",
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_changed }}",
"rightValue": ""
}
]
},
"renameOutput": true
},
{
"outputKey": "file_added",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_added }}",
"rightValue": ""
}
]
},
"renameOutput": true
}
]
},
"options": {}
},
"typeVersion": 3
},
{
"id": "da91b2aa-613c-4e3e-af83-fbd3bb7e922e",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
123.92779403575491
],
"parameters": {
"color": 7,
"width": 847.032584995578,
"height": 335.8400964393443,
"content": "## Step 2. When files are removed, the vector point is cleared.\n[Learn how to delete points using the Qdrant API](https://qdrant.tech/documentation/concepts/points/#delete-points)\n\nTo keep our vectorstore relevant, we'll implement a simple synchronisation system whereby documents deleted from the local file folder are also purged from Qdrant. This can be simply achieved using Qdrant APIs."
},
"typeVersion": 1
},
{
"id": "2f9f5b2b-6504-4b27-a0c4-f3373df352df",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
480
],
"parameters": {
"color": 7,
"width": 855.9952607674757,
"height": 433.01782147687817,
"content": "## Step 3. When files are updated, the vector point is updated.\n[Learn how to delete points using the Qdrant API](https://qdrant.tech/documentation/concepts/points/#delete-points)\n\nSimilarly to the files deleted branch, when we encounter a change in a file we'll update the matching vector point in Qdrant to ensure our vector store stays relevant. Here, we can achieve this my deleting the existing vector point and creating it anew with the updated bank statement."
},
"typeVersion": 1
},
{
"id": "38128b7f-d0f2-405c-a7de-662df812c344",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
940
],
"parameters": {
"color": 7,
"width": 846.8204626627492,
"height": 629.9714759033081,
"content": "## Step 4. When new files are added, add them to Qdrant Vectorstore.\n[Read more about the Qdrant node](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreqdrant)\n\nUsing Qdrant, we'll able to create a simple yet powerful RAG based application for our bank statements. One of Qdrant's most powerful features is its filtering system, we'll use it to manage the synchronisation of our local file system and Qdrant."
},
"typeVersion": 1
},
{
"id": "e85e2a30-e775-42fe-a12a-ac5de4eb4673",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2180,
491.43199269284935
],
"parameters": {
"color": 7,
"width": 744.4578330639196,
"height": 759.7908149448928,
"content": "## Step 5. Create AI Agent expert on historic bank statements \n[Read more about the Question & Answer Chain](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.chainretrievalqa)\n\nFinally, let's use a Question & Answer AI node to combine the Mistral AI model and Qdrant as the vector store retriever to create a local expert for all our bank statements questions. "
},
"typeVersion": 1
},
{
"id": "7b29b0b9-ffee-4456-b036-9b39400d2b31",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
1700,
1120
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "={{ $('Set Variables').item.json.qdrant_collection }}"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
},
{
"id": "1857bebb-b492-415e-96c8-235329bfd28a",
"name": "Qdrant Vector Store1",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
2620,
960
],
"parameters": {
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "BankStatements"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Read File": {
"main": [
[
{
"node": "Prepare Embedding Document",
"type": "main",
"index": 0
}
]
]
},
"Chat Trigger": {
"main": [
[
{
"node": "Question and Answer Chain",
"type": "main",
"index": 0
}
]
]
},
"Set Variables": {
"main": [
[
{
"node": "Handle File Event",
"type": "main",
"index": 0
}
]
]
},
"Handle File Event": {
"main": [
[
{
"node": "Search For Existing Point",
"type": "main",
"index": 0
}
],
[
{
"node": "Search For Existing Point1",
"type": "main",
"index": 0
}
],
[
{
"node": "Read File",
"type": "main",
"index": 0
}
]
]
},
"Local File Trigger": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Has Existing Point?": {
"main": [
[
{
"node": "Delete Existing Point1",
"type": "main",
"index": 0
}
]
]
},
"Has Existing Point?1": {
"main": [
[
{
"node": "Delete Existing Point",
"type": "main",
"index": 0
}
]
]
},
"Qdrant Vector Store1": {
"ai_vectorStore": [
[
{
"node": "Vector Store Retriever",
"type": "ai_vectorStore",
"index": 0
}
]
]
},
"Delete Existing Point": {
"main": [
[
{
"node": "Remap for File_Added Flow",
"type": "main",
"index": 0
}
]
]
},
"Vector Store Retriever": {
"ai_retriever": [
[
{
"node": "Question and Answer Chain",
"type": "ai_retriever",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Mistral Cloud Chat Model": {
"ai_languageModel": [
[
{
"node": "Question and Answer Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud1": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store1",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Remap for File_Added Flow": {
"main": [
[
{
"node": "Read File",
"type": "main",
"index": 0
}
]
]
},
"Search For Existing Point": {
"main": [
[
{
"node": "Has Existing Point?",
"type": "main",
"index": 0
}
]
]
},
"Prepare Embedding Document": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
}
]
]
},
"Search For Existing Point1": {
"main": [
[
{
"node": "Has Existing Point?1",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}

940
AI_Research_RAG_and_Data_Analysis/Build a Financial Documents Assistant using Qdrant and Mistral.ai.txt
View File

@@ -1,940 +0,0 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "c5525f47-4d91-4b98-87bb-566b90da64a1",
"name": "Local File Trigger",
"type": "n8n-nodes-base.localFileTrigger",
"position": [
660,
700
],
"parameters": {
"path": "/home/node/host_mount/local_file_search",
"events": [
"add",
"change",
"unlink"
],
"options": {
"awaitWriteFinish": true
},
"triggerOn": "folder"
},
"typeVersion": 1
},
{
"id": "804334d6-e34d-40d1-9555-b331ffe66f6f",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
664.5766613599001,
881.8474780113352
],
"parameters": {},
"typeVersion": 1
},
{
"id": "7ab0e284-b667-4d1f-8ceb-fb05e4081a06",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
840,
700
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "35ea70c4-8669-4975-a68d-bbaa094713c0",
"name": "directory",
"type": "string",
"value": "/home/node/BankStatements"
},
{
"id": "1d081d19-ff4e-462a-9cbe-7af2244bf87f",
"name": "file_added",
"type": "string",
"value": "={{ $json.event === 'add' && $json.path || ''}}"
},
{
"id": "18f8dc03-51ca-48c7-947f-87ce8e1979bf",
"name": "file_changed",
"type": "string",
"value": "={{ $json.event === 'change' && $json.path || '' }}"
},
{
"id": "65074ff7-037b-4b3b-b2c3-8a61755ab43b",
"name": "file_deleted",
"type": "string",
"value": "={{ $json.event === 'unlink' && $json.path || '' }}"
},
{
"id": "9a1902e7-f94d-4d1f-9006-91c67354d3e8",
"name": "qdrant_collection",
"type": "string",
"value": "local_file_search"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "76173972-ceca-43a4-b85f-00b41f774304",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
580,
460
],
"parameters": {
"color": 7,
"width": 665.0909497859384,
"height": 596.8351502261468,
"content": "## Step 1. Select the target folder\n[Read more about local file trigger](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.localfiletrigger)\n\nIn this workflow, we'll monitor a specific folder on disk that n8n has access to. Since we're using docker, we can either use the n8n volume or mount a folder from the host machine.\n\nThe local file trigger is useful to execute the workflow whenever changes are made to our target folder."
},
"typeVersion": 1
},
{
"id": "eda839f7-dde4-4d1f-9fe6-692df4ac7282",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
184.57666135990007,
461.84747801133517
],
"parameters": {
"width": 372.51107341403605,
"height": 356.540665091993,
"content": "## Try It Out!\n### This workflow does the following:\n* Monitors a target folder for changes using the local file trigger\n* Synchronises files in the target folder with their vectors in Qdrant\n* Mistral AI is used to create a Q&A AI agent on all files in the target folder\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "f82f6de0-af8f-4fdf-a733-f59ba4fed02f",
"name": "Read File",
"type": "n8n-nodes-base.readWriteFile",
"position": [
1340,
1120
],
"parameters": {
"options": {},
"fileSelector": "={{ $json.file_added }}"
},
"typeVersion": 1
},
{
"id": "7354a080-051b-479f-97b1-49cc0c14c9d8",
"name": "Embeddings Mistral Cloud",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
1720,
1280
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "a1ad45ff-a882-4aed-82e2-cad2483cf4e8",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
1820,
1280
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "filter_by_filename",
"value": "={{ $json.file_location }}"
},
{
"name": "filter_by_created_month",
"value": "={{ $now.year + '-' + $now.monthShort }}"
},
{
"name": "filter_by_created_week",
"value": "={{ $now.year + '-' + $now.monthShort + '-W' + $now.weekNumber }}"
}
]
}
},
"jsonData": "={{ $json.data }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "0b0e29b9-8873-4074-94dc-9f0364c28835",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
1840,
1400
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "c0555ba6-a1bd-4aa9-a340-a9c617f8e6db",
"name": "Prepare Embedding Document",
"type": "n8n-nodes-base.set",
"position": [
1520,
1120
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "41a1d4ca-e5a5-4fb9-b249-8796ae759b33",
"name": "data",
"type": "string",
"value": "=## file location\n{{ [$json.directory, $json.fileName].join('/') }}\n## file created\n{{ $now.toISO() }}\n## file contents\n{{ $input.item.binary.data.data.base64Decode() }}"
},
{
"id": "c091704d-b81c-448b-8c90-156ef568b871",
"name": "file_location",
"type": "string",
"value": "={{ [$json.directory, $json.fileName].join('/') }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "ffe8c363-0809-4d21-aa8f-34b0fc2dc57f",
"name": "Chat Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
2280,
680
],
"webhookId": "37587fe0-b8db-4012-90a7-1f65b9bfd0df",
"parameters": {},
"typeVersion": 1
},
{
"id": "8d958669-60be-4bb2-80fc-2a6c7c7bfae6",
"name": "Question and Answer Chain",
"type": "@n8n/n8n-nodes-langchain.chainRetrievalQa",
"position": [
2500,
680
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "f143e438-8176-4923-a866-3f9a2a16793d",
"name": "Mistral Cloud Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatMistralCloud",
"position": [
2500,
840
],
"parameters": {
"model": "mistral-small-2402",
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "06dd8f4c-3b66-43e0-85c8-ec222e275f87",
"name": "Vector Store Retriever",
"type": "@n8n/n8n-nodes-langchain.retrieverVectorStore",
"position": [
2620,
840
],
"parameters": {},
"typeVersion": 1
},
{
"id": "2fdabcb5-a7a7-4e02-8c1b-9190e2e52385",
"name": "Embeddings Mistral Cloud1",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
2620,
1080
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "e5664534-de07-481f-87dd-68d7d0715baa",
"name": "Remap for File_Added Flow",
"type": "n8n-nodes-base.set",
"position": [
1920,
700
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "840219e1-ed47-4b00-83fd-6b3c0bd71650",
"name": "file_added",
"type": "string",
"value": "={{ $('Set Variables').item.json.file_changed }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "1fd14832-aafe-4d72-b4f2-7afc72df97dc",
"name": "Search For Existing Point",
"type": "n8n-nodes-base.httpRequest",
"position": [
1340,
280
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/scroll",
"method": "POST",
"options": {},
"jsonBody": "={\n \"filter\": {\n \"must\": [\n {\n \"key\": \"metadata.filter_by_filename\",\n \"match\": {\n \"value\": \"{{ $json.file_changed }}\"\n }\n }\n ]\n },\n \"limit\": 1,\n \"with_payload\": false,\n \"with_vector\": false\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "b5fa817f-82d6-41dd-9817-4c1dd9137b76",
"name": "Has Existing Point?",
"type": "n8n-nodes-base.if",
"position": [
1520,
280
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0392bac0-8fb5-406b-b59f-575edf5ab30d",
"operator": {
"type": "array",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.result.points }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "b0fa4fa4-5d1b-4a12-b8ba-a10d71f31f94",
"name": "Delete Existing Point",
"type": "n8n-nodes-base.httpRequest",
"position": [
1720,
700
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/delete",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "points",
"value": "={{ $json.result.points.map(point => point.id) }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "5408adfe-4d6b-407c-aac7-e87c9b1a1592",
"name": "Search For Existing Point1",
"type": "n8n-nodes-base.httpRequest",
"position": [
1340,
700
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/scroll",
"method": "POST",
"options": {},
"jsonBody": "={\n \"filter\": {\n \"must\": [\n {\n \"key\": \"metadata.filter_by_filename\",\n \"match\": {\n \"value\": \"{{ $json.file_changed }}\"\n }\n }\n ]\n },\n \"limit\": 1,\n \"with_payload\": false,\n \"with_vector\": false\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "fac43587-0d24-4d6e-a0d5-8cc8f9615967",
"name": "Has Existing Point?1",
"type": "n8n-nodes-base.if",
"position": [
1520,
700
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0392bac0-8fb5-406b-b59f-575edf5ab30d",
"operator": {
"type": "array",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.result.points }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "010baacd-fac1-4cc1-86bf-9d6ef11916fe",
"name": "Delete Existing Point1",
"type": "n8n-nodes-base.httpRequest",
"position": [
1700,
280
],
"parameters": {
"url": "=http://qdrant:6333/collections/{{ $('Set Variables').item.json.qdrant_collection }}/points/delete",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "points",
"value": "={{ $json.result.points.map(point => point.id) }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "2d6fb29c-2fac-41de-9ad0-cc781b246378",
"name": "Handle File Event",
"type": "n8n-nodes-base.switch",
"position": [
1000,
700
],
"parameters": {
"rules": {
"values": [
{
"outputKey": "file_deleted",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "a1f6d86a-9805-4d0e-ac70-90c9cf0ad339",
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_deleted }}",
"rightValue": ""
}
]
},
"renameOutput": true
},
{
"outputKey": "file_changed",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "d15cde67-b5b0-4676-b4fb-ead749147392",
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_changed }}",
"rightValue": ""
}
]
},
"renameOutput": true
},
{
"outputKey": "file_added",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"operator": {
"type": "string",
"operation": "notEmpty",
"singleValue": true
},
"leftValue": "={{ $json.file_added }}",
"rightValue": ""
}
]
},
"renameOutput": true
}
]
},
"options": {}
},
"typeVersion": 3
},
{
"id": "da91b2aa-613c-4e3e-af83-fbd3bb7e922e",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
123.92779403575491
],
"parameters": {
"color": 7,
"width": 847.032584995578,
"height": 335.8400964393443,
"content": "## Step 2. When files are removed, the vector point is cleared.\n[Learn how to delete points using the Qdrant API](https://qdrant.tech/documentation/concepts/points/#delete-points)\n\nTo keep our vectorstore relevant, we'll implement a simple synchronisation system whereby documents deleted from the local file folder are also purged from Qdrant. This can be simply achieved using Qdrant APIs."
},
"typeVersion": 1
},
{
"id": "2f9f5b2b-6504-4b27-a0c4-f3373df352df",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
480
],
"parameters": {
"color": 7,
"width": 855.9952607674757,
"height": 433.01782147687817,
"content": "## Step 3. When files are updated, the vector point is updated.\n[Learn how to delete points using the Qdrant API](https://qdrant.tech/documentation/concepts/points/#delete-points)\n\nSimilarly to the files deleted branch, when we encounter a change in a file we'll update the matching vector point in Qdrant to ensure our vector store stays relevant. Here, we can achieve this my deleting the existing vector point and creating it anew with the updated bank statement."
},
"typeVersion": 1
},
{
"id": "38128b7f-d0f2-405c-a7de-662df812c344",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1280,
940
],
"parameters": {
"color": 7,
"width": 846.8204626627492,
"height": 629.9714759033081,
"content": "## Step 4. When new files are added, add them to Qdrant Vectorstore.\n[Read more about the Qdrant node](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreqdrant)\n\nUsing Qdrant, we'll able to create a simple yet powerful RAG based application for our bank statements. One of Qdrant's most powerful features is its filtering system, we'll use it to manage the synchronisation of our local file system and Qdrant."
},
"typeVersion": 1
},
{
"id": "e85e2a30-e775-42fe-a12a-ac5de4eb4673",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2180,
491.43199269284935
],
"parameters": {
"color": 7,
"width": 744.4578330639196,
"height": 759.7908149448928,
"content": "## Step 5. Create AI Agent expert on historic bank statements \n[Read more about the Question & Answer Chain](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.chainretrievalqa)\n\nFinally, let's use a Question & Answer AI node to combine the Mistral AI model and Qdrant as the vector store retriever to create a local expert for all our bank statements questions. "
},
"typeVersion": 1
},
{
"id": "7b29b0b9-ffee-4456-b036-9b39400d2b31",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
1700,
1120
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "={{ $('Set Variables').item.json.qdrant_collection }}"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
},
{
"id": "1857bebb-b492-415e-96c8-235329bfd28a",
"name": "Qdrant Vector Store1",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
2620,
960
],
"parameters": {
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "BankStatements"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Read File": {
"main": [
[
{
"node": "Prepare Embedding Document",
"type": "main",
"index": 0
}
]
]
},
"Chat Trigger": {
"main": [
[
{
"node": "Question and Answer Chain",
"type": "main",
"index": 0
}
]
]
},
"Set Variables": {
"main": [
[
{
"node": "Handle File Event",
"type": "main",
"index": 0
}
]
]
},
"Handle File Event": {
"main": [
[
{
"node": "Search For Existing Point",
"type": "main",
"index": 0
}
],
[
{
"node": "Search For Existing Point1",
"type": "main",
"index": 0
}
],
[
{
"node": "Read File",
"type": "main",
"index": 0
}
]
]
},
"Local File Trigger": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Has Existing Point?": {
"main": [
[
{
"node": "Delete Existing Point1",
"type": "main",
"index": 0
}
]
]
},
"Has Existing Point?1": {
"main": [
[
{
"node": "Delete Existing Point",
"type": "main",
"index": 0
}
]
]
},
"Qdrant Vector Store1": {
"ai_vectorStore": [
[
{
"node": "Vector Store Retriever",
"type": "ai_vectorStore",
"index": 0
}
]
]
},
"Delete Existing Point": {
"main": [
[
{
"node": "Remap for File_Added Flow",
"type": "main",
"index": 0
}
]
]
},
"Vector Store Retriever": {
"ai_retriever": [
[
{
"node": "Question and Answer Chain",
"type": "ai_retriever",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Mistral Cloud Chat Model": {
"ai_languageModel": [
[
{
"node": "Question and Answer Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud1": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store1",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Remap for File_Added Flow": {
"main": [
[
{
"node": "Read File",
"type": "main",
"index": 0
}
]
]
},
"Search For Existing Point": {
"main": [
[
{
"node": "Has Existing Point?",
"type": "main",
"index": 0
}
]
]
},
"Prepare Embedding Document": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
}
]
]
},
"Search For Existing Point1": {
"main": [
[
{
"node": "Has Existing Point?1",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}

1147
AI_Research_RAG_and_Data_Analysis/Build a Tax Code Assistant with Qdrant, Mistral.ai and OpenAI.json Normal file
View File

File diff suppressed because it is too large Load Diff

1147
AI_Research_RAG_and_Data_Analysis/Build a Tax Code Assistant with Qdrant, Mistral.ai and OpenAI.txt
View File

File diff suppressed because it is too large Load Diff

849
AI_Research_RAG_and_Data_Analysis/Building RAG Chatbot for Movie Recommendations with Qdrant and Open AI.json Normal file
View File

@@ -0,0 +1,849 @@
{
"id": "a58HZKwcOy7lmz56",
"meta": {
"instanceId": "178ef8a5109fc76c716d40bcadb720c455319f7b7a3fd5a39e4f336a091f524a",
"templateCredsSetupCompleted": true
},
"name": "Building RAG Chatbot for Movie Recommendations with Qdrant and Open AI",
"tags": [],
"nodes": [
{
"id": "06a34e3b-519a-4b48-afd0-4f2b51d2105d",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
4980,
740
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9213003d-433f-41ab-838b-be93860261b2",
"name": "GitHub",
"type": "n8n-nodes-base.github",
"position": [
5200,
740
],
"parameters": {
"owner": {
"__rl": true,
"mode": "name",
"value": "mrscoopers"
},
"filePath": "Top_1000_IMDB_movies.csv",
"resource": "file",
"operation": "get",
"repository": {
"__rl": true,
"mode": "list",
"value": "n8n_demo",
"cachedResultUrl": "https://github.com/mrscoopers/n8n_demo",
"cachedResultName": "n8n_demo"
},
"additionalParameters": {}
},
"credentials": {
"githubApi": {
"id": "VbfC0mqEq24vPIwq",
"name": "GitHub n8n demo"
}
},
"typeVersion": 1
},
{
"id": "9850d1a9-3a6f-44c0-9f9d-4d20fda0b602",
"name": "Extract from File",
"type": "n8n-nodes-base.extractFromFile",
"position": [
5360,
740
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "7704f993-b1c9-477a-8b5a-77dc2cb68161",
"name": "Embeddings OpenAI",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
5560,
940
],
"parameters": {
"model": "text-embedding-3-small",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 1
},
{
"id": "bc6dd8e5-0186-4bf9-9c60-2eab6d9b6520",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
5700,
960
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "movie_name",
"value": "={{ $('Extract from File').item.json['Movie Name'] }}"
},
{
"name": "movie_release_date",
"value": "={{ $('Extract from File').item.json['Year of Release'] }}"
},
{
"name": "movie_description",
"value": "={{ $('Extract from File').item.json.Description }}"
}
]
}
},
"jsonData": "={{ $('Extract from File').item.json.Description }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "f87ea014-fe79-444b-88ea-0c4773872b0a",
"name": "Token Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterTokenSplitter",
"position": [
5700,
1140
],
"parameters": {},
"typeVersion": 1
},
{
"id": "d8d28cec-c8e8-4350-9e98-cdbc6da54988",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
5600,
740
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "imdb"
}
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 1
},
{
"id": "f86e03dc-12ea-4929-9035-4ec3cf46e300",
"name": "When chat message received",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
4920,
1140
],
"webhookId": "71bfe0f8-227e-466b-9d07-69fd9fe4a27b",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "ead23ef6-2b6b-428d-b412-b3394bff8248",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
5040,
1340
],
"parameters": {
"model": "gpt-4o-mini",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 1
},
{
"id": "7ab936e1-aac8-43bc-a497-f2d02c2c19e5",
"name": "Call n8n Workflow Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
5320,
1340
],
"parameters": {
"name": "movie_recommender",
"schemaType": "manual",
"workflowId": {
"__rl": true,
"mode": "id",
"value": "a58HZKwcOy7lmz56"
},
"description": "Call this tool to get a list of recommended movies from a vector database. ",
"inputSchema": "{\n\"type\": \"object\",\n\"properties\": {\n\t\"positive_example\": {\n \"type\": \"string\",\n \"description\": \"A string with a movie description matching the user's positive recommendation request\"\n },\n \"negative_example\": {\n \"type\": \"string\",\n \"description\": \"A string with a movie description matching the user's negative anti-recommendation reuqest\"\n }\n}\n}",
"specifyInputSchema": true
},
"typeVersion": 1.2
},
{
"id": "ce55f334-698b-45b1-9e12-0eaa473187d4",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
5160,
1340
],
"parameters": {},
"typeVersion": 1.2
},
{
"id": "41c1ee11-3117-4765-98fc-e56cc6fc8fb2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
5640,
1600
],
"parameters": {},
"typeVersion": 1
},
{
"id": "db8d6ab6-8cd2-4a8c-993d-f1b7d7fdcffd",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
6540,
1500
],
"parameters": {
"mode": "combine",
"options": {},
"combineBy": "combineAll"
},
"typeVersion": 3
},
{
"id": "c7bc5e04-22b1-40db-ba74-1ab234e51375",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
7260,
1480
],
"parameters": {
"options": {},
"fieldToSplitOut": "result"
},
"typeVersion": 1
},
{
"id": "a2002d2e-362a-49eb-a42d-7b665ddd67a0",
"name": "Split Out1",
"type": "n8n-nodes-base.splitOut",
"position": [
7140,
1260
],
"parameters": {
"options": {},
"fieldToSplitOut": "result.points"
},
"typeVersion": 1
},
{
"id": "f69a87f1-bfb9-4337-9350-28d2416c1580",
"name": "Merge1",
"type": "n8n-nodes-base.merge",
"position": [
7520,
1400
],
"parameters": {
"mode": "combine",
"options": {},
"fieldsToMatchString": "id"
},
"typeVersion": 3
},
{
"id": "b2f2529e-e260-4d72-88ef-09b804226004",
"name": "Aggregate",
"type": "n8n-nodes-base.aggregate",
"position": [
7960,
1400
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData",
"destinationFieldName": "response"
},
"typeVersion": 1
},
{
"id": "bedea10f-b4de-4f0e-9d60-cc8117a2b328",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
5140,
1140
],
"parameters": {
"options": {
"systemMessage": "You are a Movie Recommender Tool using a Vector Database under the hood. Provide top-3 movie recommendations returned by the database, ordered by their recommendation score, but not showing the score to the user."
}
},
"typeVersion": 1.6
},
{
"id": "e04276b5-7d69-437b-bf4f-9717808cc8f6",
"name": "Embedding Recommendation Request with Open AI",
"type": "n8n-nodes-base.httpRequest",
"position": [
5900,
1460
],
"parameters": {
"url": "https://api.openai.com/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "input",
"value": "={{ $json.query.positive_example }}"
},
{
"name": "model",
"value": "text-embedding-3-small"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Authorization",
"value": "Bearer $OPENAI_API_KEY"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 4.2
},
{
"id": "68e99f06-82f5-432c-8b31-8a1ae34981a6",
"name": "Embedding Anti-Recommendation Request with Open AI",
"type": "n8n-nodes-base.httpRequest",
"position": [
5920,
1660
],
"parameters": {
"url": "https://api.openai.com/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "input",
"value": "={{ $json.query.negative_example }}"
},
{
"name": "model",
"value": "text-embedding-3-small"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Authorization",
"value": "Bearer $OPENAI_API_KEY"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ecb1d7e1-b389-48e8-a34a-176bfc923641",
"name": "Extracting Embedding",
"type": "n8n-nodes-base.set",
"position": [
6180,
1460
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "01a28c9d-aeb1-48bb-8a73-f8bddbd73460",
"name": "positive_example",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "4ed11142-a734-435f-9f7a-f59e2d423076",
"name": "Extracting Embedding1",
"type": "n8n-nodes-base.set",
"position": [
6180,
1660
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "01a28c9d-aeb1-48bb-8a73-f8bddbd73460",
"name": "negative_example",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "ce3aa9bc-a5b1-4529-bff5-e0dba43b99f3",
"name": "Calling Qdrant Recommendation API",
"type": "n8n-nodes-base.httpRequest",
"position": [
6840,
1500
],
"parameters": {
"url": "https://edcc6735-2ffb-484f-b735-3467043828fe.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/imdb_1000_open_ai/points/query",
"method": "POST",
"options": {},
"jsonBody": "={\n \"query\": {\n \"recommend\": {\n \"positive\": [[{{ $json.positive_example }}]],\n \"negative\": [[{{ $json.negative_example }}]],\n \"strategy\": \"average_vector\"\n }\n },\n \"limit\":3\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 4.2
},
{
"id": "9b8a6bdb-16fe-4edc-86d0-136fe059a777",
"name": "Retrieving Recommended Movies Meta Data",
"type": "n8n-nodes-base.httpRequest",
"position": [
7060,
1460
],
"parameters": {
"url": "https://edcc6735-2ffb-484f-b735-3467043828fe.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/imdb_1000_open_ai/points",
"method": "POST",
"options": {},
"jsonBody": "={\n \"ids\": [\"{{ $json.result.points[0].id }}\", \"{{ $json.result.points[1].id }}\", \"{{ $json.result.points[2].id }}\"],\n \"with_payload\":true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 4.2
},
{
"id": "28cdcad5-3dca-48a1-b626-19eef657114c",
"name": "Selecting Fields Relevant for Agent",
"type": "n8n-nodes-base.set",
"position": [
7740,
1400
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "b4b520a5-d0e2-4dcb-af9d-0b7748fd44d6",
"name": "movie_recommendation_score",
"type": "number",
"value": "={{ $json.score }}"
},
{
"id": "c9f0982e-bd4e-484b-9eab-7e69e333f706",
"name": "movie_description",
"type": "string",
"value": "={{ $json.payload.content }}"
},
{
"id": "7c7baf11-89cd-4695-9f37-13eca7e01163",
"name": "movie_name",
"type": "string",
"value": "={{ $json.payload.metadata.movie_name }}"
},
{
"id": "1d1d269e-43c7-47b0-859b-268adf2dbc21",
"name": "movie_release_year",
"type": "string",
"value": "={{ $json.payload.metadata.release_year }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "56e73f01-5557-460a-9a63-01357a1b456f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
5560,
1780
],
"parameters": {
"content": "Tool, calling Qdrant's recommendation API based on user's request, transformed by AI agent"
},
"typeVersion": 1
},
{
"id": "cce5250e-0285-4fd0-857f-4b117151cd8b",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
4680,
720
],
"parameters": {
"content": "Uploading data (movies and their descriptions) to Qdrant Vector Store\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"negative_example": "horror bloody movie",
"positive_example": "romantic comedy"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "40d3669b-d333-435f-99fc-db623deda2cb",
"connections": {
"Merge": {
"main": [
[
{
"node": "Calling Qdrant Recommendation API",
"type": "main",
"index": 0
}
]
]
},
"GitHub": {
"main": [
[
{
"node": "Extract from File",
"type": "main",
"index": 0
}
]
]
},
"Merge1": {
"main": [
[
{
"node": "Selecting Fields Relevant for Agent",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Merge1",
"type": "main",
"index": 1
}
]
]
},
"Split Out1": {
"main": [
[
{
"node": "Merge1",
"type": "main",
"index": 0
}
]
]
},
"Token Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
},
"Embeddings OpenAI": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Extract from File": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Extracting Embedding": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "AI Agent",
"type": "ai_memory",
"index": 0
}
]
]
},
"Extracting Embedding1": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Call n8n Workflow Tool": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Embedding Recommendation Request with Open AI",
"type": "main",
"index": 0
},
{
"node": "Embedding Anti-Recommendation Request with Open AI",
"type": "main",
"index": 0
}
]
]
},
"When chat message received": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Calling Qdrant Recommendation API": {
"main": [
[
{
"node": "Retrieving Recommended Movies Meta Data",
"type": "main",
"index": 0
},
{
"node": "Split Out1",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "GitHub",
"type": "main",
"index": 0
}
]
]
},
"Selecting Fields Relevant for Agent": {
"main": [
[
{
"node": "Aggregate",
"type": "main",
"index": 0
}
]
]
},
"Retrieving Recommended Movies Meta Data": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Embedding Recommendation Request with Open AI": {
"main": [
[
{
"node": "Extracting Embedding",
"type": "main",
"index": 0
}
]
]
},
"Embedding Anti-Recommendation Request with Open AI": {
"main": [
[
{
"node": "Extracting Embedding1",
"type": "main",
"index": 0
}
]
]
}
}
}

849
AI_Research_RAG_and_Data_Analysis/Building RAG Chatbot for Movie Recommendations with Qdrant and Open AI.txt
View File

@@ -1,849 +0,0 @@
{
"id": "a58HZKwcOy7lmz56",
"meta": {
"instanceId": "178ef8a5109fc76c716d40bcadb720c455319f7b7a3fd5a39e4f336a091f524a",
"templateCredsSetupCompleted": true
},
"name": "Building RAG Chatbot for Movie Recommendations with Qdrant and Open AI",
"tags": [],
"nodes": [
{
"id": "06a34e3b-519a-4b48-afd0-4f2b51d2105d",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
4980,
740
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9213003d-433f-41ab-838b-be93860261b2",
"name": "GitHub",
"type": "n8n-nodes-base.github",
"position": [
5200,
740
],
"parameters": {
"owner": {
"__rl": true,
"mode": "name",
"value": "mrscoopers"
},
"filePath": "Top_1000_IMDB_movies.csv",
"resource": "file",
"operation": "get",
"repository": {
"__rl": true,
"mode": "list",
"value": "n8n_demo",
"cachedResultUrl": "https://github.com/mrscoopers/n8n_demo",
"cachedResultName": "n8n_demo"
},
"additionalParameters": {}
},
"credentials": {
"githubApi": {
"id": "VbfC0mqEq24vPIwq",
"name": "GitHub n8n demo"
}
},
"typeVersion": 1
},
{
"id": "9850d1a9-3a6f-44c0-9f9d-4d20fda0b602",
"name": "Extract from File",
"type": "n8n-nodes-base.extractFromFile",
"position": [
5360,
740
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "7704f993-b1c9-477a-8b5a-77dc2cb68161",
"name": "Embeddings OpenAI",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
5560,
940
],
"parameters": {
"model": "text-embedding-3-small",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 1
},
{
"id": "bc6dd8e5-0186-4bf9-9c60-2eab6d9b6520",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
5700,
960
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "movie_name",
"value": "={{ $('Extract from File').item.json['Movie Name'] }}"
},
{
"name": "movie_release_date",
"value": "={{ $('Extract from File').item.json['Year of Release'] }}"
},
{
"name": "movie_description",
"value": "={{ $('Extract from File').item.json.Description }}"
}
]
}
},
"jsonData": "={{ $('Extract from File').item.json.Description }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "f87ea014-fe79-444b-88ea-0c4773872b0a",
"name": "Token Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterTokenSplitter",
"position": [
5700,
1140
],
"parameters": {},
"typeVersion": 1
},
{
"id": "d8d28cec-c8e8-4350-9e98-cdbc6da54988",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
5600,
740
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "id",
"value": "imdb"
}
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 1
},
{
"id": "f86e03dc-12ea-4929-9035-4ec3cf46e300",
"name": "When chat message received",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
4920,
1140
],
"webhookId": "71bfe0f8-227e-466b-9d07-69fd9fe4a27b",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "ead23ef6-2b6b-428d-b412-b3394bff8248",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
5040,
1340
],
"parameters": {
"model": "gpt-4o-mini",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 1
},
{
"id": "7ab936e1-aac8-43bc-a497-f2d02c2c19e5",
"name": "Call n8n Workflow Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
5320,
1340
],
"parameters": {
"name": "movie_recommender",
"schemaType": "manual",
"workflowId": {
"__rl": true,
"mode": "id",
"value": "a58HZKwcOy7lmz56"
},
"description": "Call this tool to get a list of recommended movies from a vector database. ",
"inputSchema": "{\n\"type\": \"object\",\n\"properties\": {\n\t\"positive_example\": {\n \"type\": \"string\",\n \"description\": \"A string with a movie description matching the user's positive recommendation request\"\n },\n \"negative_example\": {\n \"type\": \"string\",\n \"description\": \"A string with a movie description matching the user's negative anti-recommendation reuqest\"\n }\n}\n}",
"specifyInputSchema": true
},
"typeVersion": 1.2
},
{
"id": "ce55f334-698b-45b1-9e12-0eaa473187d4",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
5160,
1340
],
"parameters": {},
"typeVersion": 1.2
},
{
"id": "41c1ee11-3117-4765-98fc-e56cc6fc8fb2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
5640,
1600
],
"parameters": {},
"typeVersion": 1
},
{
"id": "db8d6ab6-8cd2-4a8c-993d-f1b7d7fdcffd",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
6540,
1500
],
"parameters": {
"mode": "combine",
"options": {},
"combineBy": "combineAll"
},
"typeVersion": 3
},
{
"id": "c7bc5e04-22b1-40db-ba74-1ab234e51375",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
7260,
1480
],
"parameters": {
"options": {},
"fieldToSplitOut": "result"
},
"typeVersion": 1
},
{
"id": "a2002d2e-362a-49eb-a42d-7b665ddd67a0",
"name": "Split Out1",
"type": "n8n-nodes-base.splitOut",
"position": [
7140,
1260
],
"parameters": {
"options": {},
"fieldToSplitOut": "result.points"
},
"typeVersion": 1
},
{
"id": "f69a87f1-bfb9-4337-9350-28d2416c1580",
"name": "Merge1",
"type": "n8n-nodes-base.merge",
"position": [
7520,
1400
],
"parameters": {
"mode": "combine",
"options": {},
"fieldsToMatchString": "id"
},
"typeVersion": 3
},
{
"id": "b2f2529e-e260-4d72-88ef-09b804226004",
"name": "Aggregate",
"type": "n8n-nodes-base.aggregate",
"position": [
7960,
1400
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData",
"destinationFieldName": "response"
},
"typeVersion": 1
},
{
"id": "bedea10f-b4de-4f0e-9d60-cc8117a2b328",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
5140,
1140
],
"parameters": {
"options": {
"systemMessage": "You are a Movie Recommender Tool using a Vector Database under the hood. Provide top-3 movie recommendations returned by the database, ordered by their recommendation score, but not showing the score to the user."
}
},
"typeVersion": 1.6
},
{
"id": "e04276b5-7d69-437b-bf4f-9717808cc8f6",
"name": "Embedding Recommendation Request with Open AI",
"type": "n8n-nodes-base.httpRequest",
"position": [
5900,
1460
],
"parameters": {
"url": "https://api.openai.com/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "input",
"value": "={{ $json.query.positive_example }}"
},
{
"name": "model",
"value": "text-embedding-3-small"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Authorization",
"value": "Bearer $OPENAI_API_KEY"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 4.2
},
{
"id": "68e99f06-82f5-432c-8b31-8a1ae34981a6",
"name": "Embedding Anti-Recommendation Request with Open AI",
"type": "n8n-nodes-base.httpRequest",
"position": [
5920,
1660
],
"parameters": {
"url": "https://api.openai.com/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "input",
"value": "={{ $json.query.negative_example }}"
},
{
"name": "model",
"value": "text-embedding-3-small"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Authorization",
"value": "Bearer $OPENAI_API_KEY"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "deYJUwkgL1Euu613",
"name": "OpenAi account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ecb1d7e1-b389-48e8-a34a-176bfc923641",
"name": "Extracting Embedding",
"type": "n8n-nodes-base.set",
"position": [
6180,
1460
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "01a28c9d-aeb1-48bb-8a73-f8bddbd73460",
"name": "positive_example",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "4ed11142-a734-435f-9f7a-f59e2d423076",
"name": "Extracting Embedding1",
"type": "n8n-nodes-base.set",
"position": [
6180,
1660
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "01a28c9d-aeb1-48bb-8a73-f8bddbd73460",
"name": "negative_example",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "ce3aa9bc-a5b1-4529-bff5-e0dba43b99f3",
"name": "Calling Qdrant Recommendation API",
"type": "n8n-nodes-base.httpRequest",
"position": [
6840,
1500
],
"parameters": {
"url": "https://edcc6735-2ffb-484f-b735-3467043828fe.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/imdb_1000_open_ai/points/query",
"method": "POST",
"options": {},
"jsonBody": "={\n \"query\": {\n \"recommend\": {\n \"positive\": [[{{ $json.positive_example }}]],\n \"negative\": [[{{ $json.negative_example }}]],\n \"strategy\": \"average_vector\"\n }\n },\n \"limit\":3\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 4.2
},
{
"id": "9b8a6bdb-16fe-4edc-86d0-136fe059a777",
"name": "Retrieving Recommended Movies Meta Data",
"type": "n8n-nodes-base.httpRequest",
"position": [
7060,
1460
],
"parameters": {
"url": "https://edcc6735-2ffb-484f-b735-3467043828fe.europe-west3-0.gcp.cloud.qdrant.io:6333/collections/imdb_1000_open_ai/points",
"method": "POST",
"options": {},
"jsonBody": "={\n \"ids\": [\"{{ $json.result.points[0].id }}\", \"{{ $json.result.points[1].id }}\", \"{{ $json.result.points[2].id }}\"],\n \"with_payload\":true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "Zin08PA0RdXVUKK7",
"name": "QdrantApi n8n demo"
}
},
"typeVersion": 4.2
},
{
"id": "28cdcad5-3dca-48a1-b626-19eef657114c",
"name": "Selecting Fields Relevant for Agent",
"type": "n8n-nodes-base.set",
"position": [
7740,
1400
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "b4b520a5-d0e2-4dcb-af9d-0b7748fd44d6",
"name": "movie_recommendation_score",
"type": "number",
"value": "={{ $json.score }}"
},
{
"id": "c9f0982e-bd4e-484b-9eab-7e69e333f706",
"name": "movie_description",
"type": "string",
"value": "={{ $json.payload.content }}"
},
{
"id": "7c7baf11-89cd-4695-9f37-13eca7e01163",
"name": "movie_name",
"type": "string",
"value": "={{ $json.payload.metadata.movie_name }}"
},
{
"id": "1d1d269e-43c7-47b0-859b-268adf2dbc21",
"name": "movie_release_year",
"type": "string",
"value": "={{ $json.payload.metadata.release_year }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "56e73f01-5557-460a-9a63-01357a1b456f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
5560,
1780
],
"parameters": {
"content": "Tool, calling Qdrant's recommendation API based on user's request, transformed by AI agent"
},
"typeVersion": 1
},
{
"id": "cce5250e-0285-4fd0-857f-4b117151cd8b",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
4680,
720
],
"parameters": {
"content": "Uploading data (movies and their descriptions) to Qdrant Vector Store\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"negative_example": "horror bloody movie",
"positive_example": "romantic comedy"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "40d3669b-d333-435f-99fc-db623deda2cb",
"connections": {
"Merge": {
"main": [
[
{
"node": "Calling Qdrant Recommendation API",
"type": "main",
"index": 0
}
]
]
},
"GitHub": {
"main": [
[
{
"node": "Extract from File",
"type": "main",
"index": 0
}
]
]
},
"Merge1": {
"main": [
[
{
"node": "Selecting Fields Relevant for Agent",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Merge1",
"type": "main",
"index": 1
}
]
]
},
"Split Out1": {
"main": [
[
{
"node": "Merge1",
"type": "main",
"index": 0
}
]
]
},
"Token Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
},
"Embeddings OpenAI": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Extract from File": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Extracting Embedding": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "AI Agent",
"type": "ai_memory",
"index": 0
}
]
]
},
"Extracting Embedding1": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Call n8n Workflow Tool": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Embedding Recommendation Request with Open AI",
"type": "main",
"index": 0
},
{
"node": "Embedding Anti-Recommendation Request with Open AI",
"type": "main",
"index": 0
}
]
]
},
"When chat message received": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Calling Qdrant Recommendation API": {
"main": [
[
{
"node": "Retrieving Recommended Movies Meta Data",
"type": "main",
"index": 0
},
{
"node": "Split Out1",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "GitHub",
"type": "main",
"index": 0
}
]
]
},
"Selecting Fields Relevant for Agent": {
"main": [
[
{
"node": "Aggregate",
"type": "main",
"index": 0
}
]
]
},
"Retrieving Recommended Movies Meta Data": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Embedding Recommendation Request with Open AI": {
"main": [
[
{
"node": "Extracting Embedding",
"type": "main",
"index": 0
}
]
]
},
"Embedding Anti-Recommendation Request with Open AI": {
"main": [
[
{
"node": "Extracting Embedding1",
"type": "main",
"index": 0
}
]
]
}
}
}

426
AI_Research_RAG_and_Data_Analysis/Chat with GitHub API Documentation_ RAG-Powered Chatbot with Pinecone & OpenAI.json Normal file
View File

@@ -0,0 +1,426 @@
{
"id": "FD0bHNaehP3LzCNN",
"meta": {
"instanceId": "69133932b9ba8e1ef14816d0b63297bb44feb97c19f759b5d153ff6b0c59e18d"
},
"name": "Chat with GitHub OpenAPI Specification using RAG (Pinecone and OpenAI)",
"tags": [],
"nodes": [
{
"id": "362cb773-7540-4753-a401-e585cdf4af8a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
0,
0
],
"parameters": {},
"typeVersion": 1
},
{
"id": "45470036-cae6-48d0-ac66-addc8999e776",
"name": "HTTP Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
300,
0
],
"parameters": {
"url": "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "a9e65897-52c9-4941-bf49-e1a659e442ef",
"name": "Pinecone Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStorePinecone",
"position": [
520,
0
],
"parameters": {
"mode": "insert",
"options": {},
"pineconeIndex": {
"__rl": true,
"mode": "list",
"value": "n8n-demo",
"cachedResultName": "n8n-demo"
}
},
"credentials": {
"pineconeApi": {
"id": "bQTNry52ypGLqt47",
"name": "PineconeApi account"
}
},
"typeVersion": 1
},
{
"id": "c2a2354b-5457-4ceb-abfc-9a58e8593b81",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
660,
180
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "7338d9ea-ae8f-46eb-807f-a15dc7639fc9",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
740,
360
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "44fd7a59-f208-4d5d-a22d-e9f8ca9badf1",
"name": "When chat message received",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
-20,
760
],
"webhookId": "089e38ab-4eee-4c34-aa5d-54cf4a8f53b7",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "51d819d6-70ff-428d-aa56-1d7e06490dee",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
320,
760
],
"parameters": {
"options": {
"systemMessage": "You are a helpful assistant providing information about the GitHub API and how to use it based on the OpenAPI V3 specifications."
}
},
"typeVersion": 1.7
},
{
"id": "aed548bf-7083-44ad-a3e0-163dee7423ef",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
220,
980
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
},
{
"id": "dfe9f356-2225-4f4b-86c7-e56a230b4193",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
420,
1020
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "4cf672ee-13b8-4355-b8e0-c2e7381671bc",
"name": "Vector Store Tool",
"type": "@n8n/n8n-nodes-langchain.toolVectorStore",
"position": [
580,
980
],
"parameters": {
"name": "GitHub_OpenAPI_Specification",
"description": "Use this tool to get information about the GitHub API. This database contains OpenAPI v3 specifications."
},
"typeVersion": 1
},
{
"id": "1df7fb85-9d4a-4db5-9bed-41d28e2e4643",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
840,
1160
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
},
{
"id": "7b52ef7a-5935-451e-8747-efe16ce288af",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-40,
-260
],
"parameters": {
"width": 640,
"height": 200,
"content": "## Indexing content in the vector database\nThis part of the workflow is responsible for extracting content, generating embeddings and sending them to the Pinecone vector store.\n\nIt requests the OpenAPI specifications from GitHub using a HTTP request. Then, it splits the file in chunks, generating embeddings for each chunk using OpenAI, and saving them in Pinecone vector DB."
},
"typeVersion": 1
},
{
"id": "3508d602-56d4-4818-84eb-ca75cdeec1d0",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-20,
560
],
"parameters": {
"width": 580,
"content": "## Querying and response generation \n\nThis part of the workflow is responsible for the chat interface, querying the vector store and generating relevant responses.\n\nIt uses OpenAI GPT 4o-mini to generate responses."
},
"typeVersion": 1
},
{
"id": "5a9808ef-4edd-4ec9-ba01-2fe50b2dbf4b",
"name": "Generate User Query Embedding",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
480,
1400
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.2
},
{
"id": "f703dc8e-9d4b-45e3-8994-789b3dfe8631",
"name": "Pinecone Vector Store (Querying)",
"type": "@n8n/n8n-nodes-langchain.vectorStorePinecone",
"position": [
440,
1220
],
"parameters": {
"options": {},
"pineconeIndex": {
"__rl": true,
"mode": "list",
"value": "n8n-demo",
"cachedResultName": "n8n-demo"
}
},
"credentials": {
"pineconeApi": {
"id": "bQTNry52ypGLqt47",
"name": "PineconeApi account"
}
},
"typeVersion": 1
},
{
"id": "ea64a7a5-1fa5-4938-83a9-271929733a8e",
"name": "Generate Embeddings",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
480,
220
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.2
},
{
"id": "65cbd4e3-91f6-441a-9ef1-528c3019e238",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-820,
-260
],
"parameters": {
"width": 620,
"height": 320,
"content": "## RAG workflow in n8n\n\nThis is an example of how to use RAG techniques to create a chatbot with n8n. It is an API documentation chatbot that can answer questions about the GitHub API. It uses OpenAI for generating embeddings, the gpt-4o-mini LLM for generating responses and Pinecone as a vector database.\n\n### Before using this template\n* create OpenAI and Pinecone accounts\n* obtain API keys OpenAI and Pinecone \n* configure credentials in n8n for both\n* ensure you have a Pinecone index named \"n8n-demo\" or adjust the workflow accordingly."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "2908105f-c20c-4183-bb9d-26e3559b9911",
"connections": {
"HTTP Request": {
"main": [
[
{
"node": "Pinecone Vector Store",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Vector Store Tool": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Vector Store Tool",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Pinecone Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Generate Embeddings": {
"ai_embedding": [
[
{
"node": "Pinecone Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "AI Agent",
"type": "ai_memory",
"index": 0
}
]
]
},
"When chat message received": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Generate User Query Embedding": {
"ai_embedding": [
[
{
"node": "Pinecone Vector Store (Querying)",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Pinecone Vector Store (Querying)": {
"ai_vectorStore": [
[
{
"node": "Vector Store Tool",
"type": "ai_vectorStore",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "HTTP Request",
"type": "main",
"index": 0
}
]
]
}
}
}

426
AI_Research_RAG_and_Data_Analysis/Chat with GitHub API Documentation_ RAG-Powered Chatbot with Pinecone & OpenAI.txt
View File

@@ -1,426 +0,0 @@
{
"id": "FD0bHNaehP3LzCNN",
"meta": {
"instanceId": "69133932b9ba8e1ef14816d0b63297bb44feb97c19f759b5d153ff6b0c59e18d"
},
"name": "Chat with GitHub OpenAPI Specification using RAG (Pinecone and OpenAI)",
"tags": [],
"nodes": [
{
"id": "362cb773-7540-4753-a401-e585cdf4af8a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
0,
0
],
"parameters": {},
"typeVersion": 1
},
{
"id": "45470036-cae6-48d0-ac66-addc8999e776",
"name": "HTTP Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
300,
0
],
"parameters": {
"url": "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "a9e65897-52c9-4941-bf49-e1a659e442ef",
"name": "Pinecone Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStorePinecone",
"position": [
520,
0
],
"parameters": {
"mode": "insert",
"options": {},
"pineconeIndex": {
"__rl": true,
"mode": "list",
"value": "n8n-demo",
"cachedResultName": "n8n-demo"
}
},
"credentials": {
"pineconeApi": {
"id": "bQTNry52ypGLqt47",
"name": "PineconeApi account"
}
},
"typeVersion": 1
},
{
"id": "c2a2354b-5457-4ceb-abfc-9a58e8593b81",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
660,
180
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "7338d9ea-ae8f-46eb-807f-a15dc7639fc9",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
740,
360
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "44fd7a59-f208-4d5d-a22d-e9f8ca9badf1",
"name": "When chat message received",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
-20,
760
],
"webhookId": "089e38ab-4eee-4c34-aa5d-54cf4a8f53b7",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "51d819d6-70ff-428d-aa56-1d7e06490dee",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
320,
760
],
"parameters": {
"options": {
"systemMessage": "You are a helpful assistant providing information about the GitHub API and how to use it based on the OpenAPI V3 specifications."
}
},
"typeVersion": 1.7
},
{
"id": "aed548bf-7083-44ad-a3e0-163dee7423ef",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
220,
980
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
},
{
"id": "dfe9f356-2225-4f4b-86c7-e56a230b4193",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
420,
1020
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "4cf672ee-13b8-4355-b8e0-c2e7381671bc",
"name": "Vector Store Tool",
"type": "@n8n/n8n-nodes-langchain.toolVectorStore",
"position": [
580,
980
],
"parameters": {
"name": "GitHub_OpenAPI_Specification",
"description": "Use this tool to get information about the GitHub API. This database contains OpenAPI v3 specifications."
},
"typeVersion": 1
},
{
"id": "1df7fb85-9d4a-4db5-9bed-41d28e2e4643",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
840,
1160
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
},
{
"id": "7b52ef7a-5935-451e-8747-efe16ce288af",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-40,
-260
],
"parameters": {
"width": 640,
"height": 200,
"content": "## Indexing content in the vector database\nThis part of the workflow is responsible for extracting content, generating embeddings and sending them to the Pinecone vector store.\n\nIt requests the OpenAPI specifications from GitHub using a HTTP request. Then, it splits the file in chunks, generating embeddings for each chunk using OpenAI, and saving them in Pinecone vector DB."
},
"typeVersion": 1
},
{
"id": "3508d602-56d4-4818-84eb-ca75cdeec1d0",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-20,
560
],
"parameters": {
"width": 580,
"content": "## Querying and response generation \n\nThis part of the workflow is responsible for the chat interface, querying the vector store and generating relevant responses.\n\nIt uses OpenAI GPT 4o-mini to generate responses."
},
"typeVersion": 1
},
{
"id": "5a9808ef-4edd-4ec9-ba01-2fe50b2dbf4b",
"name": "Generate User Query Embedding",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
480,
1400
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.2
},
{
"id": "f703dc8e-9d4b-45e3-8994-789b3dfe8631",
"name": "Pinecone Vector Store (Querying)",
"type": "@n8n/n8n-nodes-langchain.vectorStorePinecone",
"position": [
440,
1220
],
"parameters": {
"options": {},
"pineconeIndex": {
"__rl": true,
"mode": "list",
"value": "n8n-demo",
"cachedResultName": "n8n-demo"
}
},
"credentials": {
"pineconeApi": {
"id": "bQTNry52ypGLqt47",
"name": "PineconeApi account"
}
},
"typeVersion": 1
},
{
"id": "ea64a7a5-1fa5-4938-83a9-271929733a8e",
"name": "Generate Embeddings",
"type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
"position": [
480,
220
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "tQLWnWRzD8aebYvp",
"name": "OpenAi account"
}
},
"typeVersion": 1.2
},
{
"id": "65cbd4e3-91f6-441a-9ef1-528c3019e238",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-820,
-260
],
"parameters": {
"width": 620,
"height": 320,
"content": "## RAG workflow in n8n\n\nThis is an example of how to use RAG techniques to create a chatbot with n8n. It is an API documentation chatbot that can answer questions about the GitHub API. It uses OpenAI for generating embeddings, the gpt-4o-mini LLM for generating responses and Pinecone as a vector database.\n\n### Before using this template\n* create OpenAI and Pinecone accounts\n* obtain API keys OpenAI and Pinecone \n* configure credentials in n8n for both\n* ensure you have a Pinecone index named \"n8n-demo\" or adjust the workflow accordingly."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "2908105f-c20c-4183-bb9d-26e3559b9911",
"connections": {
"HTTP Request": {
"main": [
[
{
"node": "Pinecone Vector Store",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Vector Store Tool": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Vector Store Tool",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Pinecone Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Generate Embeddings": {
"ai_embedding": [
[
{
"node": "Pinecone Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "AI Agent",
"type": "ai_memory",
"index": 0
}
]
]
},
"When chat message received": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Generate User Query Embedding": {
"ai_embedding": [
[
{
"node": "Pinecone Vector Store (Querying)",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Pinecone Vector Store (Querying)": {
"ai_vectorStore": [
[
{
"node": "Vector Store Tool",
"type": "ai_vectorStore",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "HTTP Request",
"type": "main",
"index": 0
}
]
]
}
}
}

690
AI_Research_RAG_and_Data_Analysis/Create a Google Analytics Data Report with AI and sent it to E-Mail and Telegram.json Normal file
View File

@@ -0,0 +1,690 @@
{
"id": "AAjX1BuwhyXpo8xP",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a"
},
"name": "Google Analytics: Weekly Report",
"tags": [],
"nodes": [
{
"id": "91ba5982-e226-4f0b-af0d-8c9a44b08279",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
-1740,
300
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
1
],
"triggerAtHour": 7
}
]
}
},
"typeVersion": 1.2
},
{
"id": "62c38eaf-2222-4d22-8589-677f36bce10d",
"name": "Google Analytics Letzte 7 Tage",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
-1540,
300
],
"parameters": {
"metricsGA4": {
"metricValues": [
{
"listName": "screenPageViews"
},
{},
{
"listName": "sessions"
},
{
"listName": "sessionsPerUser"
},
{
"name": "averageSessionDuration",
"listName": "other"
},
{
"name": "ecommercePurchases",
"listName": "other"
},
{
"name": "averagePurchaseRevenue",
"listName": "other"
},
{
"name": "purchaseRevenue",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "list",
"value": "345060083",
"cachedResultUrl": "https://analytics.google.com/analytics/web/#/p345060083/",
"cachedResultName": "https://www.ep-reisen.de  – GA4"
},
"dimensionsGA4": {
"dimensionValues": [
{}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "onRKXREI8izfGzv0",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "0a51c2f3-a487-4226-884f-63d4cb2bf4e4",
"name": "Send Email",
"type": "n8n-nodes-base.emailSend",
"position": [
420,
80
],
"parameters": {
"html": "={{ $json.message.content }}",
"options": {},
"subject": "Weekly Report: Google Analytics: Last 7 days",
"toEmail": "friedemann.schuetz@ep-reisen.de",
"fromEmail": "friedemann.schuetz@posteo.de"
},
"credentials": {
"smtp": {
"id": "A71x7hx6lKj7nxp1",
"name": "SMTP account"
}
},
"typeVersion": 2.1
},
{
"id": "04963783-f455-4983-afea-e94b316d8532",
"name": "Telegram",
"type": "n8n-nodes-base.telegram",
"position": [
420,
420
],
"parameters": {
"text": "={{ $json.message.content }}",
"chatId": "1810565648",
"additionalFields": {}
},
"credentials": {
"telegramApi": {
"id": "0hnyvxyUMN77sBmU",
"name": "Telegram account"
}
},
"typeVersion": 1.2
},
{
"id": "3b6b4902-15b3-4bbc-8427-c35471a7431b",
"name": "Processing for Telegram",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
60,
420
],
"parameters": {
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini",
"cachedResultName": "GPT-4O-MINI"
},
"options": {},
"messages": {
"values": [
{
"content": "=Convert the following text from HTML to normal text:\n\n{{ $json.message.content }}\n\nPlease format the table so that each metric is a separate paragraph!\n\nExample:\n\nTotal views: xx.xxx\nTotal views previous year: xx,xxx\nDifference: x.xx %\n\nTotal users: xx,xxx\nTotal users previous year: xx,xxx\nDifference: -x.xx %"
}
]
}
},
"credentials": {
"openAiApi": {
"id": "niikB3HA4fT5WAqt",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
},
{
"id": "d761980c-0327-4d4e-92aa-d0342b2e249e",
"name": "Calculator",
"type": "@n8n/n8n-nodes-langchain.toolCalculator",
"position": [
140,
300
],
"parameters": {},
"typeVersion": 1
},
{
"id": "ce7ba356-80bb-4b17-9445-fb535267cdf0",
"name": "Google Analytics: Past 7 days of the previous year",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
-600,
300
],
"parameters": {
"endDate": "={{ $json.endDate }}",
"dateRange": "custom",
"startDate": "={{ $json.startDate }}",
"metricsGA4": {
"metricValues": [
{
"listName": "screenPageViews"
},
{},
{
"listName": "sessions"
},
{
"listName": "sessionsPerUser"
},
{
"name": "averageSessionDuration",
"listName": "other"
},
{
"name": "ecommercePurchases",
"listName": "other"
},
{
"name": "averagePurchaseRevenue",
"listName": "other"
},
{
"name": "purchaseRevenue",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "list",
"value": "345060083",
"cachedResultUrl": "https://analytics.google.com/analytics/web/#/p345060083/",
"cachedResultName": "https://www.ep-reisen.de  – GA4"
},
"dimensionsGA4": {
"dimensionValues": [
{}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "onRKXREI8izfGzv0",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "d2062aaa-e41b-4405-8470-9e7b4cd77245",
"name": "Summarize Data",
"type": "n8n-nodes-base.summarize",
"position": [
-1080,
300
],
"parameters": {
"options": {},
"fieldsToSummarize": {
"values": [
{
"field": "Aufrufe",
"aggregation": "sum"
},
{
"field": "Nutzer",
"aggregation": "sum"
},
{
"field": "Sitzungen",
"aggregation": "sum"
},
{
"field": "Sitzungen pro Nutzer",
"aggregation": "average"
},
{
"field": "Sitzungsdauer",
"aggregation": "average"
},
{
"field": "Käufe",
"aggregation": "sum"
},
{
"field": "Revenue pro Kauf",
"aggregation": "average"
},
{
"field": "Revenue",
"aggregation": "sum"
},
{
"field": "date"
}
]
}
},
"typeVersion": 1
},
{
"id": "d1f48d36-9f27-4cda-af53-e6d430d1a8db",
"name": "Summarize Data1",
"type": "n8n-nodes-base.summarize",
"position": [
-220,
300
],
"parameters": {
"options": {},
"fieldsToSummarize": {
"values": [
{
"field": "Aufrufe",
"aggregation": "sum"
},
{
"field": "Nutzer",
"aggregation": "sum"
},
{
"field": "Sitzungen",
"aggregation": "sum"
},
{
"field": "Sitzungen pro Nutzer",
"aggregation": "average"
},
{
"field": "Sitzungsdauer",
"aggregation": "average"
},
{
"field": "Käufe",
"aggregation": "sum"
},
{
"field": "Revenue pro Kauf",
"aggregation": "average"
},
{
"field": "Revenue",
"aggregation": "sum"
},
{
"field": "date"
}
]
}
},
"typeVersion": 1
},
{
"id": "5b6a0644-3839-4a62-8ff3-bf866aa4568c",
"name": "Calculation same period previous year",
"type": "n8n-nodes-base.code",
"position": [
-840,
300
],
"parameters": {
"jsCode": "return {\n // Berechnung des Startdatums: Vorjahr, gleiche Woche, 7 Tage zurück\n startDate: (() => {\n const date = new Date();\n date.setFullYear(date.getFullYear() - 1); // Zurück ins Vorjahr\n date.setDate(date.getDate() - 7); // 7 Tage zurück\n return date.toISOString().split('T')[0];\n })(),\n \n // Berechnung des Enddatums: Vorjahr, heutiges Datum\n endDate: (() => {\n const date = new Date();\n date.setFullYear(date.getFullYear() - 1); // Zurück ins Vorjahr\n return date.toISOString().split('T')[0];\n })(),\n};\n"
},
"typeVersion": 2
},
{
"id": "ab813532-cbe6-4c41-b20b-7efaa1ae4389",
"name": "Assign data",
"type": "n8n-nodes-base.set",
"position": [
-1300,
300
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9c2f8b9a-e964-49a0-8837-efb0dfd7bcae",
"name": "Aufrufe",
"type": "number",
"value": "={{ $json.screenPageViews }}"
},
{
"id": "8b524518-1268-4971-b5c9-ae7da09d94f9",
"name": "Nutzer",
"type": "number",
"value": "={{ $json.totalUsers }}"
},
{
"id": "ca7279b9-c643-425f-aa99-cb17146e9994",
"name": "Sitzungen",
"type": "number",
"value": "={{ $json.sessions }}"
},
{
"id": "591288f7-e8cf-445e-872a-5b83f997b825",
"name": "Sitzungen pro Nutzer",
"type": "number",
"value": "={{ $json.sessionsPerUser }}"
},
{
"id": "dc1a43da-3f3a-4dca-bbde-904222d7f693",
"name": "Sitzungsdauer",
"type": "number",
"value": "={{ $json.averageSessionDuration }}"
},
{
"id": "eac0b53e-c452-40b8-92bc-8af8ea349984",
"name": "=Käufe",
"type": "number",
"value": "={{ $json.ecommercePurchases }}"
},
{
"id": "b96439be-189d-4ebe-b49e-d5c31fefe9f0",
"name": "Revenue pro Kauf",
"type": "number",
"value": "={{ $json.averagePurchaseRevenue }}"
},
{
"id": "94835d43-2fc8-49c0-97f0-6f0f8699337a",
"name": "Revenue",
"type": "number",
"value": "={{ $json.purchaseRevenue }}"
},
{
"id": "d70f8138-3b84-4b88-a98f-eb929e1cc29a",
"name": "date",
"type": "string",
"value": "={{ $json.date }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "2454fe8a-005d-46dc-ae22-1044c1b793b7",
"name": "Assign data1",
"type": "n8n-nodes-base.set",
"position": [
-400,
300
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9c2f8b9a-e964-49a0-8837-efb0dfd7bcae",
"name": "Aufrufe",
"type": "number",
"value": "={{ $json.screenPageViews }}"
},
{
"id": "8b524518-1268-4971-b5c9-ae7da09d94f9",
"name": "Nutzer",
"type": "number",
"value": "={{ $json.totalUsers }}"
},
{
"id": "ca7279b9-c643-425f-aa99-cb17146e9994",
"name": "Sitzungen",
"type": "number",
"value": "={{ $json.sessions }}"
},
{
"id": "591288f7-e8cf-445e-872a-5b83f997b825",
"name": "Sitzungen pro Nutzer",
"type": "number",
"value": "={{ $json.sessionsPerUser }}"
},
{
"id": "dc1a43da-3f3a-4dca-bbde-904222d7f693",
"name": "Sitzungsdauer",
"type": "number",
"value": "={{ $json.averageSessionDuration }}"
},
{
"id": "eac0b53e-c452-40b8-92bc-8af8ea349984",
"name": "=Käufe",
"type": "number",
"value": "={{ $json.ecommercePurchases }}"
},
{
"id": "b96439be-189d-4ebe-b49e-d5c31fefe9f0",
"name": "Revenue pro Kauf",
"type": "number",
"value": "={{ $json.averagePurchaseRevenue }}"
},
{
"id": "94835d43-2fc8-49c0-97f0-6f0f8699337a",
"name": "Revenue",
"type": "number",
"value": "={{ $json.purchaseRevenue }}"
},
{
"id": "dd8255c6-65b1-41ce-b596-70c09108d6e2",
"name": "=date",
"type": "string",
"value": "={{ $json.date }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "0a48cbb0-3d4c-4ac8-8dba-08213f7fc430",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-2220,
80
],
"parameters": {
"width": 440,
"height": 560,
"content": "Welcome to my Google Analytics Weekly Report Workflow!\n\nThis workflow has the following sequence:\n\n1. time trigger (e.g. every Monday at 7 a.m.)\n2. retrieval of Google Analytics data from the last 7 days\n3. assignment and summary of the data\n4. retrieval of Google Analytics data from the last 7 days of the previous year\n5. allocation and summary of the data\n6. preparation in tabular form and brief analysis by AI.\n7. sending the report as an email\n8. preparation in short form by AI for Telegram (optional)\n9. sending as Telegram message.\n\nThe following accesses are required for the workflow:\n- Google Analytics (via Google Analytics API): https://docs.n8n.io/integrations/builtin/credentials/google/\n- AI API access (e.g. via OpenAI, Anthropic, Google or Ollama)\n- SMTP access data (for sending the mail)\n- Telegram access data (optional for sending as Telegram message): https://docs.n8n.io/integrations/builtin/credentials/telegram/\n\nYou can contact me via LinkedIn, if you have any questions: https://www.linkedin.com/in/friedemann-schuetz"
},
"typeVersion": 1
},
{
"id": "c87bc648-8fe8-4cec-84d4-2742060f9c53",
"name": "Processing for email",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
60,
80
],
"parameters": {
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o",
"cachedResultName": "GPT-4O"
},
"options": {},
"messages": {
"values": [
{
"content": "=Please analyze the following data and output the results in tabular form:\n\n| Metrics | Last 7 days | Previous year | Percentage change |\n|-------------------------------|---------------|---------|\n| Total page views | {{ $('Summarize Data').item.json.sum_Aufrufe }} | {{ $('Summarize Data1').item.json.sum_Aufrufe }} | Percentage change |\n| total users | {{ $('Summarize Data').item.json.sum_Nutzer }} | {{ $('Summarize Data1').item.json.sum_Nutzer }} | Percentage change |\n| Total sessions | {{ $('Summarize Data').item.json.sum_Sitzungen }} | {{ $('Summarize Data1').item.json.sum_Sitzungen }} | Percentage change |\n| Average sessions/user | {{ $('Summarize Data').item.json.average_Sitzungen_pro_Nutzer }} | {{ $('Summarize Data1').item.json.average_Sitzungen_pro_Nutzer }} | Percentage change |\n| Average session duration | {{ $('Summarize Data').item.json.average_Sitzungsdauer }} | {{ $('Summarize Data1').item.json.average_Sitzungsdauer }} | Percentage change |\n| Total purchases | {{ $('Summarize Data').item.json['sum_Käufe'] }} | {{ $('Summarize Data1').item.json['sum_Käufe'] }} | Percentage change |\n| Average revenue/purchase | {{ $('Summarize Data').item.json.average_Revenue_pro_Kauf }} | {{ $('Summarize Data1').item.json.average_Revenue_pro_Kauf }} | Percentage change |\n| Total revenue | {{ $('Summarize Data').item.json.sum_Revenue }} | {{ $('Summarize Data1').item.json.sum_Revenue }} | Percentage change |\n\nFormat for numbers:\n- Dot (.) for numbers in thousands (e.g. 4,000)\n- Comma (,) for decimal numbers (e.g. 3.4)\n- Conversion of average session duration in minutes instead of seconds\n- Average turnover/purchase and total turnover in €\n\nPlease write a short summary of the analyzed data above the table (in a maximum of 3 sentences!)\n\nPlease format to a sleek and modern HTML format so that the result can be sent as HTML mail!\n\nStructure of the e-mail:\n\n“Hello! Here is the Weekly Report: Google Analytics of the last 7 days!\n[Summary]\n[Table]”"
}
]
}
},
"credentials": {
"openAiApi": {
"id": "niikB3HA4fT5WAqt",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "556c3292-0d40-4c75-8037-90bacf1b2ccb",
"connections": {
"Telegram": {
"main": [
[]
]
},
"Calculator": {
"ai_tool": [
[
{
"node": "Processing for email",
"type": "ai_tool",
"index": 0
}
]
]
},
"Assign data": {
"main": [
[
{
"node": "Summarize Data",
"type": "main",
"index": 0
}
]
]
},
"Assign data1": {
"main": [
[
{
"node": "Summarize Data1",
"type": "main",
"index": 0
}
]
]
},
"Summarize Data": {
"main": [
[
{
"node": "Calculation same period previous year",
"type": "main",
"index": 0
}
]
]
},
"Summarize Data1": {
"main": [
[
{
"node": "Processing for email",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Google Analytics Letzte 7 Tage",
"type": "main",
"index": 0
}
]
]
},
"Processing for email": {
"main": [
[
{
"node": "Send Email",
"type": "main",
"index": 0
},
{
"node": "Processing for Telegram",
"type": "main",
"index": 0
}
]
]
},
"Processing for Telegram": {
"main": [
[
{
"node": "Telegram",
"type": "main",
"index": 0
}
]
]
},
"Google Analytics Letzte 7 Tage": {
"main": [
[
{
"node": "Assign data",
"type": "main",
"index": 0
}
]
]
},
"Calculation same period previous year": {
"main": [
[
{
"node": "Google Analytics: Past 7 days of the previous year",
"type": "main",
"index": 0
}
]
]
},
"Google Analytics: Past 7 days of the previous year": {
"main": [
[
{
"node": "Assign data1",
"type": "main",
"index": 0
}
]
]
}
}
}

690
AI_Research_RAG_and_Data_Analysis/Create a Google Analytics Data Report with AI and sent it to E-Mail and Telegram.txt
View File

@@ -1,690 +0,0 @@
{
"id": "AAjX1BuwhyXpo8xP",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a"
},
"name": "Google Analytics: Weekly Report",
"tags": [],
"nodes": [
{
"id": "91ba5982-e226-4f0b-af0d-8c9a44b08279",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
-1740,
300
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
1
],
"triggerAtHour": 7
}
]
}
},
"typeVersion": 1.2
},
{
"id": "62c38eaf-2222-4d22-8589-677f36bce10d",
"name": "Google Analytics Letzte 7 Tage",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
-1540,
300
],
"parameters": {
"metricsGA4": {
"metricValues": [
{
"listName": "screenPageViews"
},
{},
{
"listName": "sessions"
},
{
"listName": "sessionsPerUser"
},
{
"name": "averageSessionDuration",
"listName": "other"
},
{
"name": "ecommercePurchases",
"listName": "other"
},
{
"name": "averagePurchaseRevenue",
"listName": "other"
},
{
"name": "purchaseRevenue",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "list",
"value": "345060083",
"cachedResultUrl": "https://analytics.google.com/analytics/web/#/p345060083/",
"cachedResultName": "https://www.ep-reisen.de  – GA4"
},
"dimensionsGA4": {
"dimensionValues": [
{}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "onRKXREI8izfGzv0",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "0a51c2f3-a487-4226-884f-63d4cb2bf4e4",
"name": "Send Email",
"type": "n8n-nodes-base.emailSend",
"position": [
420,
80
],
"parameters": {
"html": "={{ $json.message.content }}",
"options": {},
"subject": "Weekly Report: Google Analytics: Last 7 days",
"toEmail": "friedemann.schuetz@ep-reisen.de",
"fromEmail": "friedemann.schuetz@posteo.de"
},
"credentials": {
"smtp": {
"id": "A71x7hx6lKj7nxp1",
"name": "SMTP account"
}
},
"typeVersion": 2.1
},
{
"id": "04963783-f455-4983-afea-e94b316d8532",
"name": "Telegram",
"type": "n8n-nodes-base.telegram",
"position": [
420,
420
],
"parameters": {
"text": "={{ $json.message.content }}",
"chatId": "1810565648",
"additionalFields": {}
},
"credentials": {
"telegramApi": {
"id": "0hnyvxyUMN77sBmU",
"name": "Telegram account"
}
},
"typeVersion": 1.2
},
{
"id": "3b6b4902-15b3-4bbc-8427-c35471a7431b",
"name": "Processing for Telegram",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
60,
420
],
"parameters": {
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini",
"cachedResultName": "GPT-4O-MINI"
},
"options": {},
"messages": {
"values": [
{
"content": "=Convert the following text from HTML to normal text:\n\n{{ $json.message.content }}\n\nPlease format the table so that each metric is a separate paragraph!\n\nExample:\n\nTotal views: xx.xxx\nTotal views previous year: xx,xxx\nDifference: x.xx %\n\nTotal users: xx,xxx\nTotal users previous year: xx,xxx\nDifference: -x.xx %"
}
]
}
},
"credentials": {
"openAiApi": {
"id": "niikB3HA4fT5WAqt",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
},
{
"id": "d761980c-0327-4d4e-92aa-d0342b2e249e",
"name": "Calculator",
"type": "@n8n/n8n-nodes-langchain.toolCalculator",
"position": [
140,
300
],
"parameters": {},
"typeVersion": 1
},
{
"id": "ce7ba356-80bb-4b17-9445-fb535267cdf0",
"name": "Google Analytics: Past 7 days of the previous year",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
-600,
300
],
"parameters": {
"endDate": "={{ $json.endDate }}",
"dateRange": "custom",
"startDate": "={{ $json.startDate }}",
"metricsGA4": {
"metricValues": [
{
"listName": "screenPageViews"
},
{},
{
"listName": "sessions"
},
{
"listName": "sessionsPerUser"
},
{
"name": "averageSessionDuration",
"listName": "other"
},
{
"name": "ecommercePurchases",
"listName": "other"
},
{
"name": "averagePurchaseRevenue",
"listName": "other"
},
{
"name": "purchaseRevenue",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "list",
"value": "345060083",
"cachedResultUrl": "https://analytics.google.com/analytics/web/#/p345060083/",
"cachedResultName": "https://www.ep-reisen.de  – GA4"
},
"dimensionsGA4": {
"dimensionValues": [
{}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "onRKXREI8izfGzv0",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "d2062aaa-e41b-4405-8470-9e7b4cd77245",
"name": "Summarize Data",
"type": "n8n-nodes-base.summarize",
"position": [
-1080,
300
],
"parameters": {
"options": {},
"fieldsToSummarize": {
"values": [
{
"field": "Aufrufe",
"aggregation": "sum"
},
{
"field": "Nutzer",
"aggregation": "sum"
},
{
"field": "Sitzungen",
"aggregation": "sum"
},
{
"field": "Sitzungen pro Nutzer",
"aggregation": "average"
},
{
"field": "Sitzungsdauer",
"aggregation": "average"
},
{
"field": "Käufe",
"aggregation": "sum"
},
{
"field": "Revenue pro Kauf",
"aggregation": "average"
},
{
"field": "Revenue",
"aggregation": "sum"
},
{
"field": "date"
}
]
}
},
"typeVersion": 1
},
{
"id": "d1f48d36-9f27-4cda-af53-e6d430d1a8db",
"name": "Summarize Data1",
"type": "n8n-nodes-base.summarize",
"position": [
-220,
300
],
"parameters": {
"options": {},
"fieldsToSummarize": {
"values": [
{
"field": "Aufrufe",
"aggregation": "sum"
},
{
"field": "Nutzer",
"aggregation": "sum"
},
{
"field": "Sitzungen",
"aggregation": "sum"
},
{
"field": "Sitzungen pro Nutzer",
"aggregation": "average"
},
{
"field": "Sitzungsdauer",
"aggregation": "average"
},
{
"field": "Käufe",
"aggregation": "sum"
},
{
"field": "Revenue pro Kauf",
"aggregation": "average"
},
{
"field": "Revenue",
"aggregation": "sum"
},
{
"field": "date"
}
]
}
},
"typeVersion": 1
},
{
"id": "5b6a0644-3839-4a62-8ff3-bf866aa4568c",
"name": "Calculation same period previous year",
"type": "n8n-nodes-base.code",
"position": [
-840,
300
],
"parameters": {
"jsCode": "return {\n // Berechnung des Startdatums: Vorjahr, gleiche Woche, 7 Tage zurück\n startDate: (() => {\n const date = new Date();\n date.setFullYear(date.getFullYear() - 1); // Zurück ins Vorjahr\n date.setDate(date.getDate() - 7); // 7 Tage zurück\n return date.toISOString().split('T')[0];\n })(),\n \n // Berechnung des Enddatums: Vorjahr, heutiges Datum\n endDate: (() => {\n const date = new Date();\n date.setFullYear(date.getFullYear() - 1); // Zurück ins Vorjahr\n return date.toISOString().split('T')[0];\n })(),\n};\n"
},
"typeVersion": 2
},
{
"id": "ab813532-cbe6-4c41-b20b-7efaa1ae4389",
"name": "Assign data",
"type": "n8n-nodes-base.set",
"position": [
-1300,
300
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9c2f8b9a-e964-49a0-8837-efb0dfd7bcae",
"name": "Aufrufe",
"type": "number",
"value": "={{ $json.screenPageViews }}"
},
{
"id": "8b524518-1268-4971-b5c9-ae7da09d94f9",
"name": "Nutzer",
"type": "number",
"value": "={{ $json.totalUsers }}"
},
{
"id": "ca7279b9-c643-425f-aa99-cb17146e9994",
"name": "Sitzungen",
"type": "number",
"value": "={{ $json.sessions }}"
},
{
"id": "591288f7-e8cf-445e-872a-5b83f997b825",
"name": "Sitzungen pro Nutzer",
"type": "number",
"value": "={{ $json.sessionsPerUser }}"
},
{
"id": "dc1a43da-3f3a-4dca-bbde-904222d7f693",
"name": "Sitzungsdauer",
"type": "number",
"value": "={{ $json.averageSessionDuration }}"
},
{
"id": "eac0b53e-c452-40b8-92bc-8af8ea349984",
"name": "=Käufe",
"type": "number",
"value": "={{ $json.ecommercePurchases }}"
},
{
"id": "b96439be-189d-4ebe-b49e-d5c31fefe9f0",
"name": "Revenue pro Kauf",
"type": "number",
"value": "={{ $json.averagePurchaseRevenue }}"
},
{
"id": "94835d43-2fc8-49c0-97f0-6f0f8699337a",
"name": "Revenue",
"type": "number",
"value": "={{ $json.purchaseRevenue }}"
},
{
"id": "d70f8138-3b84-4b88-a98f-eb929e1cc29a",
"name": "date",
"type": "string",
"value": "={{ $json.date }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "2454fe8a-005d-46dc-ae22-1044c1b793b7",
"name": "Assign data1",
"type": "n8n-nodes-base.set",
"position": [
-400,
300
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "9c2f8b9a-e964-49a0-8837-efb0dfd7bcae",
"name": "Aufrufe",
"type": "number",
"value": "={{ $json.screenPageViews }}"
},
{
"id": "8b524518-1268-4971-b5c9-ae7da09d94f9",
"name": "Nutzer",
"type": "number",
"value": "={{ $json.totalUsers }}"
},
{
"id": "ca7279b9-c643-425f-aa99-cb17146e9994",
"name": "Sitzungen",
"type": "number",
"value": "={{ $json.sessions }}"
},
{
"id": "591288f7-e8cf-445e-872a-5b83f997b825",
"name": "Sitzungen pro Nutzer",
"type": "number",
"value": "={{ $json.sessionsPerUser }}"
},
{
"id": "dc1a43da-3f3a-4dca-bbde-904222d7f693",
"name": "Sitzungsdauer",
"type": "number",
"value": "={{ $json.averageSessionDuration }}"
},
{
"id": "eac0b53e-c452-40b8-92bc-8af8ea349984",
"name": "=Käufe",
"type": "number",
"value": "={{ $json.ecommercePurchases }}"
},
{
"id": "b96439be-189d-4ebe-b49e-d5c31fefe9f0",
"name": "Revenue pro Kauf",
"type": "number",
"value": "={{ $json.averagePurchaseRevenue }}"
},
{
"id": "94835d43-2fc8-49c0-97f0-6f0f8699337a",
"name": "Revenue",
"type": "number",
"value": "={{ $json.purchaseRevenue }}"
},
{
"id": "dd8255c6-65b1-41ce-b596-70c09108d6e2",
"name": "=date",
"type": "string",
"value": "={{ $json.date }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "0a48cbb0-3d4c-4ac8-8dba-08213f7fc430",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-2220,
80
],
"parameters": {
"width": 440,
"height": 560,
"content": "Welcome to my Google Analytics Weekly Report Workflow!\n\nThis workflow has the following sequence:\n\n1. time trigger (e.g. every Monday at 7 a.m.)\n2. retrieval of Google Analytics data from the last 7 days\n3. assignment and summary of the data\n4. retrieval of Google Analytics data from the last 7 days of the previous year\n5. allocation and summary of the data\n6. preparation in tabular form and brief analysis by AI.\n7. sending the report as an email\n8. preparation in short form by AI for Telegram (optional)\n9. sending as Telegram message.\n\nThe following accesses are required for the workflow:\n- Google Analytics (via Google Analytics API): https://docs.n8n.io/integrations/builtin/credentials/google/\n- AI API access (e.g. via OpenAI, Anthropic, Google or Ollama)\n- SMTP access data (for sending the mail)\n- Telegram access data (optional for sending as Telegram message): https://docs.n8n.io/integrations/builtin/credentials/telegram/\n\nYou can contact me via LinkedIn, if you have any questions: https://www.linkedin.com/in/friedemann-schuetz"
},
"typeVersion": 1
},
{
"id": "c87bc648-8fe8-4cec-84d4-2742060f9c53",
"name": "Processing for email",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
60,
80
],
"parameters": {
"modelId": {
"__rl": true,
"mode": "list",
"value": "gpt-4o",
"cachedResultName": "GPT-4O"
},
"options": {},
"messages": {
"values": [
{
"content": "=Please analyze the following data and output the results in tabular form:\n\n| Metrics | Last 7 days | Previous year | Percentage change |\n|-------------------------------|---------------|---------|\n| Total page views | {{ $('Summarize Data').item.json.sum_Aufrufe }} | {{ $('Summarize Data1').item.json.sum_Aufrufe }} | Percentage change |\n| total users | {{ $('Summarize Data').item.json.sum_Nutzer }} | {{ $('Summarize Data1').item.json.sum_Nutzer }} | Percentage change |\n| Total sessions | {{ $('Summarize Data').item.json.sum_Sitzungen }} | {{ $('Summarize Data1').item.json.sum_Sitzungen }} | Percentage change |\n| Average sessions/user | {{ $('Summarize Data').item.json.average_Sitzungen_pro_Nutzer }} | {{ $('Summarize Data1').item.json.average_Sitzungen_pro_Nutzer }} | Percentage change |\n| Average session duration | {{ $('Summarize Data').item.json.average_Sitzungsdauer }} | {{ $('Summarize Data1').item.json.average_Sitzungsdauer }} | Percentage change |\n| Total purchases | {{ $('Summarize Data').item.json['sum_Käufe'] }} | {{ $('Summarize Data1').item.json['sum_Käufe'] }} | Percentage change |\n| Average revenue/purchase | {{ $('Summarize Data').item.json.average_Revenue_pro_Kauf }} | {{ $('Summarize Data1').item.json.average_Revenue_pro_Kauf }} | Percentage change |\n| Total revenue | {{ $('Summarize Data').item.json.sum_Revenue }} | {{ $('Summarize Data1').item.json.sum_Revenue }} | Percentage change |\n\nFormat for numbers:\n- Dot (.) for numbers in thousands (e.g. 4,000)\n- Comma (,) for decimal numbers (e.g. 3.4)\n- Conversion of average session duration in minutes instead of seconds\n- Average turnover/purchase and total turnover in €\n\nPlease write a short summary of the analyzed data above the table (in a maximum of 3 sentences!)\n\nPlease format to a sleek and modern HTML format so that the result can be sent as HTML mail!\n\nStructure of the e-mail:\n\n“Hello! Here is the Weekly Report: Google Analytics of the last 7 days!\n[Summary]\n[Table]”"
}
]
}
},
"credentials": {
"openAiApi": {
"id": "niikB3HA4fT5WAqt",
"name": "OpenAi account"
}
},
"typeVersion": 1.7
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "556c3292-0d40-4c75-8037-90bacf1b2ccb",
"connections": {
"Telegram": {
"main": [
[]
]
},
"Calculator": {
"ai_tool": [
[
{
"node": "Processing for email",
"type": "ai_tool",
"index": 0
}
]
]
},
"Assign data": {
"main": [
[
{
"node": "Summarize Data",
"type": "main",
"index": 0
}
]
]
},
"Assign data1": {
"main": [
[
{
"node": "Summarize Data1",
"type": "main",
"index": 0
}
]
]
},
"Summarize Data": {
"main": [
[
{
"node": "Calculation same period previous year",
"type": "main",
"index": 0
}
]
]
},
"Summarize Data1": {
"main": [
[
{
"node": "Processing for email",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Google Analytics Letzte 7 Tage",
"type": "main",
"index": 0
}
]
]
},
"Processing for email": {
"main": [
[
{
"node": "Send Email",
"type": "main",
"index": 0
},
{
"node": "Processing for Telegram",
"type": "main",
"index": 0
}
]
]
},
"Processing for Telegram": {
"main": [
[
{
"node": "Telegram",
"type": "main",
"index": 0
}
]
]
},
"Google Analytics Letzte 7 Tage": {
"main": [
[
{
"node": "Assign data",
"type": "main",
"index": 0
}
]
]
},
"Calculation same period previous year": {
"main": [
[
{
"node": "Google Analytics: Past 7 days of the previous year",
"type": "main",
"index": 0
}
]
]
},
"Google Analytics: Past 7 days of the previous year": {
"main": [
[
{
"node": "Assign data1",
"type": "main",
"index": 0
}
]
]
}
}
}

1150
AI_Research_RAG_and_Data_Analysis/Customer Insights with Qdrant, Python and Information Extractor.json Normal file
View File

File diff suppressed because it is too large Load Diff

1150
AI_Research_RAG_and_Data_Analysis/Customer Insights with Qdrant, Python and Information Extractor.txt
View File

File diff suppressed because it is too large Load Diff

891
AI_Research_RAG_and_Data_Analysis/Deduplicate Scraping AI Grants for Eligibility using AI.json Normal file
View File

File diff suppressed because one or more lines are too long

891
AI_Research_RAG_and_Data_Analysis/Deduplicate Scraping AI Grants for Eligibility using AI.txt
View File

File diff suppressed because one or more lines are too long

960
AI_Research_RAG_and_Data_Analysis/Enrich Property Inventory Survey with Image Recognition and AI Agent.json Normal file
View File

@@ -0,0 +1,960 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "192d3e4f-6bb0-4b87-a1fa-e32c9efb49cc",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
336,
34
],
"parameters": {},
"typeVersion": 1
},
{
"id": "32a7a772-76a6-4614-a6ab-d2b152a5811f",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
1220,
180
],
"parameters": {
"model": "gpt-4o",
"options": {
"temperature": 0
}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "8c444314-ed7d-4ca0-b0fa-b6d1e964c698",
"name": "Get Applicable Rows",
"type": "n8n-nodes-base.airtable",
"position": [
516,
34
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appbgxPBurOmQK3E7",
"cachedResultUrl": "https://airtable.com/appbgxPBurOmQK3E7",
"cachedResultName": "Building Inventory Survey Example"
},
"table": {
"__rl": true,
"mode": "id",
"value": "tblEHkoTvKpa4Aa0Q"
},
"options": {},
"operation": "search",
"returnAll": false,
"filterByFormula": "AND(Image!=\"\", AI_status=FALSE())"
},
"credentials": {
"airtableTokenApi": {
"id": "Und0frCQ6SNVX3VV",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2
},
{
"id": "f90578fa-b886-4653-8ff7-0c91884dc517",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
1257,
733
],
"parameters": {},
"typeVersion": 1
},
{
"id": "8f5959eb-45bd-4185-a959-10268827e41d",
"name": "Edit Fields",
"type": "n8n-nodes-base.set",
"position": [
1417,
733
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "7263764b-8409-4cea-8db3-3278dd7ef9d8",
"name": "=route",
"type": "string",
"value": "={{ $json.route }}"
},
{
"id": "55c3b207-2e98-4137-8413-f72cbff17986",
"name": "query",
"type": "string",
"value": "={{ $json.query }}"
},
{
"id": "6eb873de-3c3a-4135-9dc0-1d441c63647c",
"name": "",
"type": "string",
"value": ""
}
]
}
},
"typeVersion": 3.3
},
{
"id": "2c7f7274-12e9-4dd3-8ee4-679b408d5430",
"name": "Fallback Response",
"type": "n8n-nodes-base.set",
"position": [
1580,
875
],
"parameters": {
"mode": "raw",
"options": {},
"jsonOutput": "{\n \"response\": {\n \"ok\": false,\n \"error\": \"The requested tool was not found or the service may be unavailable. Do not retry.\"\n }\n}\n"
},
"typeVersion": 3.3
},
{
"id": "09f36f4d-eb88-4d93-a8b3-e9ba66b46b54",
"name": "SERP Google Reverse Image API",
"type": "n8n-nodes-base.httpRequest",
"position": [
1860,
549
],
"parameters": {
"url": "https://serpapi.com/search.json",
"options": {},
"sendQuery": true,
"authentication": "predefinedCredentialType",
"queryParameters": {
"parameters": [
{
"name": "engine",
"value": "google_reverse_image"
},
{
"name": "image_url",
"value": "={{ $json.query }}"
}
]
},
"nodeCredentialType": "serpApi"
},
"credentials": {
"serpApi": {
"id": "aJCKjxx6U3K7ydDe",
"name": "SerpAPI account"
}
},
"typeVersion": 4.2
},
{
"id": "8e3a0f38-8663-4f5c-837f-4b9aa21f14fb",
"name": "Reverse Image Search Response",
"type": "n8n-nodes-base.set",
"position": [
2037,
547
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de99a504-713f-4c78-8679-08139b2def31",
"name": "response",
"type": "string",
"value": "={{ JSON.stringify($json.image_results.map(x => ({ position: x.position, title: x.title, link: x.link, description: x.snippet }))) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "0cd2269a-5b1f-4f10-b180-7f9cff9b1102",
"name": "Reverse Image Search Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
1300,
340
],
"parameters": {
"name": "reverse_image_search",
"fields": {
"values": [
{
"name": "route",
"stringValue": "serp.google_reverse_image"
}
]
},
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to perform a reverse image search. Reverse image searches return urls where similar looking products exists. Fetch the returned urls to gather more information. This tool requires the following object request body.\n```\n{\n \"type\": \"object\",\n \"properties\": {\n \"image_url\": { \"type\": \"string\" },\n }\n}\n```\nimage_url should be an absolute URL to the image."
},
"typeVersion": 1.1
},
{
"id": "9825651e-b382-4e0a-97ef-37764cb5be9e",
"name": "Firecrawl Scrape API",
"type": "n8n-nodes-base.httpRequest",
"position": [
1860,
889
],
"parameters": {
"url": "https://api.firecrawl.dev/v0/scrape",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "genericCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "url",
"value": "={{ $json.query }}"
}
]
},
"genericAuthType": "httpHeaderAuth",
"headerParameters": {
"parameters": [
{
"name": "Content-Type",
"value": "application/json"
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "OUOnyTkL9vHZNorB",
"name": "Firecrawl API"
}
},
"typeVersion": 4.2
},
{
"id": "7f61d60b-b052-4b7c-abfd-9eb8e05a45a2",
"name": "Scrape Success?",
"type": "n8n-nodes-base.if",
"position": [
2020,
889
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "a15a164f-d0c5-478f-8b27-f3d51746c214",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
},
"leftValue": "={{ $json.success }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "29c65ef4-6350-490a-b8e3-a5c869e656b2",
"name": "Firecrawl Scrape Success Response",
"type": "n8n-nodes-base.set",
"position": [
2180,
889
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "7db5c81f-de90-40e1-8086-3f13d40451c7",
"name": "response",
"type": "string",
"value": "={{ $json.data.markdown.substring(0, 3000) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "229b4008-d8a8-4609-854a-fc244a4ed630",
"name": "Firecrawl scrape Error Response",
"type": "n8n-nodes-base.set",
"position": [
2180,
1049
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "e691d86a-d366-44a2-baa6-3dba42527f6e",
"name": "response",
"type": "string",
"value": "{ error: \"Unable to scrape website due to unknown error. Do not retry.\" }"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "f080069b-e849-45e0-88cf-03707d22c704",
"name": "Firecrawl Web Scaper Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
1440,
340
],
"parameters": {
"name": "webpage_url_scraper_tool",
"fields": {
"values": [
{
"name": "route",
"stringValue": "firecrawl.scrape"
}
]
},
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to retrieve page contents of a url.\n```\n{\n \"type\": \"object\",\n \"properties\": {\n \"url\": { \"type\": \"string\" },\n }\n}\n```\nurl should be an absolute URL."
},
"typeVersion": 1.1
},
{
"id": "4eff88bb-bd5e-4d6a-b5e1-8521632c461f",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
1500,
180
],
"parameters": {
"jsonSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"title\": { \"type\": \"string\" },\n \"description\": { \"type\": \"string\" },\n \"model\": { \"type\": \"string\" },\n \"material\": { \"type\": \"string\" },\n \"color\": { \"type\": \"string\" },\n \"condition\": { \"type\": \"string\" }\n }\n}"
},
"typeVersion": 1.1
},
{
"id": "328d106b-a473-4f54-82fd-55c30d813da9",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
280,
-260
],
"parameters": {
"color": 7,
"width": 402.5984702109446,
"height": 495.4071184783251,
"content": "## 1. Use Airtable to Capture Survey Photos\n[Read more about AirTable](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.airtable)\n\nTo enable this workflow, we need a database where we can retreive the title and photo to analyse and write the generate values back to. Airtable is perfect for this since it has a robust API we can work with.\n\nFor this demo, we'll manually trigger but this can be changed for forms or other triggers."
},
"typeVersion": 1
},
{
"id": "e358775d-ff83-411d-9364-b43c87d98134",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
716.3106363781314,
-160
],
"parameters": {
"color": 7,
"width": 359.40869874940336,
"height": 428.4787925736586,
"content": "## 2. Use AI Vision Model to Analyse the Photo.\n[Read more about OpenAI Vision](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-langchain.openai)\n\nWe'll use OpenAi vision model to create a detailed description of the product in the photo. We split this step from the agent because it uses an image model rather than the usual text-based one."
},
"typeVersion": 1
},
{
"id": "51b4a70c-9583-4e8a-8e8d-896a80ad53c3",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1111.3914848823072,
-293.9250474768817
],
"parameters": {
"color": 7,
"width": 593.0683948010671,
"height": 803.956942672397,
"content": "## 3. Build an AI Agent who Searches the Internet\n[Read more about OpenAI Agents](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-langchain.openai)\n\nThis AI Agent has the ability to perform reverse image searches using our captured photos as well visit external webpages in order to obtain accurate product names and attributes. The Agent along with the tools might mimic what the average human user would carry out the same task.\n\n* For reverse image search, we're using SERP API service however we won't use the built-in SERP node as we need to specify custom parameters. \n* For scraping, we'll use [Firecrawl](https://www.firecrawl.dev/) as this service also helps to parse and return the page as markdown which is more efficient."
},
"typeVersion": 1
},
{
"id": "adfb519b-a5c7-432c-be32-5acfcc388b49",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1740,
-149.28190375244515
],
"parameters": {
"color": 7,
"width": 373.3601237414979,
"height": 397.7168664109706,
"content": "## 4. Overwrite our Rows with Enriched Results\n\nAnd Viola! Our AI agent has potentially saved hours of manual data entry work for our surveyor. This technique can be used for many other usecases."
},
"typeVersion": 1
},
{
"id": "6444e217-b944-450e-892a-5822d4d390ce",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
1200,
549
],
"parameters": {
"color": 7,
"width": 554.6092633638649,
"height": 490.7010880746526,
"content": "## 5. Using the Custom Workflow Tool\n[Read more about Workflow Tools](https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolworkflow)\n\nAI Agents rely on Tools to make decisions and become exponentially more powerful the more tools they have. A common pattern to manage multiple tools is to create a routing system for tools using the API pattern."
},
"typeVersion": 1
},
{
"id": "bf2459cf-a931-4232-9504-b36b15721194",
"name": "Enrich Product Rows",
"type": "n8n-nodes-base.airtable",
"position": [
1880,
60
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appbgxPBurOmQK3E7",
"cachedResultUrl": "https://airtable.com/appbgxPBurOmQK3E7",
"cachedResultName": "Building Inventory Survey Example"
},
"table": {
"__rl": true,
"mode": "id",
"value": "tblEHkoTvKpa4Aa0Q"
},
"columns": {
"value": {
"id": "={{ $('Get Applicable Rows').item.json.id }}",
"Color": "={{ $json.output.output.color }}",
"Model": "={{ $json.output.output.model }}",
"Title": "={{ $json.output.output.title }}",
"Material": "={{ $json.output.output.material }}",
"AI_status": true,
"Condition": "={{ $json.output.output.condition }}",
"Description": "={{ $json.output.output.description }}"
},
"schema": [
{
"id": "id",
"type": "string",
"display": true,
"removed": false,
"readOnly": true,
"required": false,
"displayName": "id",
"defaultMatch": true
},
{
"id": "Title",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Title",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Image",
"type": "array",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Image",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Description",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Description",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Model",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Model",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Material",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Material",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Color",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Color",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Condition",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Condition",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "AI_status",
"type": "boolean",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "AI_status",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"id"
]
},
"options": {},
"operation": "update"
},
"credentials": {
"airtableTokenApi": {
"id": "Und0frCQ6SNVX3VV",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2
},
{
"id": "19d736bf-c29d-46a2-93bc-b536ff28c4b5",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
-100,
-260
],
"parameters": {
"width": 359.6648027457353,
"height": 381.0536322713287,
"content": "## Try It Out!\n### This workflow does the following:\n* Scans an Airtable spreadsheet for rows with product photo images.\n* Uses an AI vision model to attempt to identify the product.\n* Uses an AI Agent to research the product on the internet to enrich the product data.\n* Overwrites our Airtable spreadsheet with the enriched data.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "25f15c48-16bf-4f92-942d-c224ed88d208",
"name": "Analyse Image",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
840,
80
],
"parameters": {
"text": "=Focus on the {{ $json.Title }} in the image - we'll refer to this as the \"object\". Identify the following attributes of the object. If you cannot determine confidently, then leave blank and move to next attribute.\n* Decription of the object.\n* The model/make of the object.\n* The material(s) used in the construction of the object.\n* The color(s) of the object\n* The condition of the object. Use one of poor, good, excellent.\n",
"options": {},
"resource": "image",
"imageUrls": "={{ $json.Image[0].thumbnails.large.url }}",
"operation": "analyze"
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1.3
},
{
"id": "e6c99f71-ccc9-426e-b916-cc38864e3224",
"name": "Object Identifier Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
1260,
20
],
"parameters": {
"text": "=system: Your role is to help an building surveyor perform a object classification and data collection task whereby the surveyor will take photos of various objects and your job is to try and identify accurately certain product attributes of the objects as detailed below.\n\nThe surveyor has given you the following:\n1) photo url ```{{ $('Get Applicable Rows').item.json.Image[0].thumbnails.large.url }}```.\n2) photo description ```{{ $json.content }}```.\n\nFor each product attribute the surveyor is unable to determine, you may:\n1) use the reverse image search tool to search the product on the internet via the provided image url.\n2) use the web scraper tool to read webpages on the internet which may be relevant to the product.\n3) If after using these tools, you are still unable to determine the required product attributes then leave the data blank.\n\nUse all the information provided and gathered, to extract the following product attributes: title, description, model, material, color and condition.",
"agent": "openAiFunctionsAgent",
"options": {},
"promptType": "define",
"hasOutputParser": true
},
"typeVersion": 1.5
},
{
"id": "661b14bd-6511-4f20-981c-2e68a7c34ec5",
"name": "Actions Router",
"type": "n8n-nodes-base.switch",
"position": [
1577,
733
],
"parameters": {
"rules": {
"values": [
{
"outputKey": "serp.google_reverse_image",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"operator": {
"type": "string",
"operation": "equals"
},
"leftValue": "={{ $json.route }}",
"rightValue": "serp.google_reverse_image"
}
]
},
"renameOutput": true
},
{
"outputKey": "firecrawl.scrape",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0a1f54ae-39f1-468d-ba6e-1376d13e4ee8",
"operator": {
"name": "filter.operator.equals",
"type": "string",
"operation": "equals"
},
"leftValue": "={{ $json.route }}",
"rightValue": "firecrawl.scrape"
}
]
},
"renameOutput": true
}
]
},
"options": {
"fallbackOutput": "extra"
}
},
"typeVersion": 3
},
{
"id": "c5078221-9239-4ec0-b25e-7cd880b58216",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
480,
20
],
"parameters": {
"width": 181.2788838920522,
"height": 297.0159375852115,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set Airtable Base and Table IDs here."
},
"typeVersion": 1
},
{
"id": "c58c0db4-9b99-4a77-90ae-66fa3981b684",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1840,
40
],
"parameters": {
"width": 181.2788838920522,
"height": 297.0159375852115,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set Airtable Base and Table IDs here."
},
"typeVersion": 1
},
{
"id": "e3a666d7-d7a5-43f5-8f04-7972332f8916",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
1780,
440
],
"parameters": {
"color": 7,
"width": 460.3301604548244,
"height": 298.81538450684064,
"content": "## 5.1 Google Reverse Image Tool\nThis tool uses Google's reverse image API to return websites where similar images are found."
},
"typeVersion": 1
},
{
"id": "d7407cdb-16bb-4bd9-a28e-7a72a5289354",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
1780,
769.9385328672522
],
"parameters": {
"color": 7,
"width": 575.3216480295998,
"height": 463.34699288922565,
"content": "## 5.2 Webscraper Tool\nThis tool uses Firecrawl.dev API to crawl webpages and returns those pages in markdown format."
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Edit Fields": {
"main": [
[
{
"node": "Actions Router",
"type": "main",
"index": 0
}
]
]
},
"Analyse Image": {
"main": [
[
{
"node": "Object Identifier Agent",
"type": "main",
"index": 0
}
]
]
},
"Actions Router": {
"main": [
[
{
"node": "SERP Google Reverse Image API",
"type": "main",
"index": 0
}
],
[
{
"node": "Firecrawl Scrape API",
"type": "main",
"index": 0
}
],
[
{
"node": "Fallback Response",
"type": "main",
"index": 0
}
]
]
},
"Scrape Success?": {
"main": [
[
{
"node": "Firecrawl Scrape Success Response",
"type": "main",
"index": 0
}
],
[
{
"node": "Firecrawl scrape Error Response",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Object Identifier Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Get Applicable Rows": {
"main": [
[
{
"node": "Analyse Image",
"type": "main",
"index": 0
}
]
]
},
"Firecrawl Scrape API": {
"main": [
[
{
"node": "Scrape Success?",
"type": "main",
"index": 0
}
]
]
},
"Object Identifier Agent": {
"main": [
[
{
"node": "Enrich Product Rows",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Edit Fields",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Object Identifier Agent",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Firecrawl Web Scaper Tool": {
"ai_tool": [
[
{
"node": "Object Identifier Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Reverse Image Search Tool": {
"ai_tool": [
[
{
"node": "Object Identifier Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"SERP Google Reverse Image API": {
"main": [
[
{
"node": "Reverse Image Search Response",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Get Applicable Rows",
"type": "main",
"index": 0
}
]
]
}
}
}

960
AI_Research_RAG_and_Data_Analysis/Enrich Property Inventory Survey with Image Recognition and AI Agent.txt
View File

@@ -1,960 +0,0 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "192d3e4f-6bb0-4b87-a1fa-e32c9efb49cc",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
336,
34
],
"parameters": {},
"typeVersion": 1
},
{
"id": "32a7a772-76a6-4614-a6ab-d2b152a5811f",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
1220,
180
],
"parameters": {
"model": "gpt-4o",
"options": {
"temperature": 0
}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "8c444314-ed7d-4ca0-b0fa-b6d1e964c698",
"name": "Get Applicable Rows",
"type": "n8n-nodes-base.airtable",
"position": [
516,
34
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appbgxPBurOmQK3E7",
"cachedResultUrl": "https://airtable.com/appbgxPBurOmQK3E7",
"cachedResultName": "Building Inventory Survey Example"
},
"table": {
"__rl": true,
"mode": "id",
"value": "tblEHkoTvKpa4Aa0Q"
},
"options": {},
"operation": "search",
"returnAll": false,
"filterByFormula": "AND(Image!=\"\", AI_status=FALSE())"
},
"credentials": {
"airtableTokenApi": {
"id": "Und0frCQ6SNVX3VV",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2
},
{
"id": "f90578fa-b886-4653-8ff7-0c91884dc517",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
1257,
733
],
"parameters": {},
"typeVersion": 1
},
{
"id": "8f5959eb-45bd-4185-a959-10268827e41d",
"name": "Edit Fields",
"type": "n8n-nodes-base.set",
"position": [
1417,
733
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "7263764b-8409-4cea-8db3-3278dd7ef9d8",
"name": "=route",
"type": "string",
"value": "={{ $json.route }}"
},
{
"id": "55c3b207-2e98-4137-8413-f72cbff17986",
"name": "query",
"type": "string",
"value": "={{ $json.query }}"
},
{
"id": "6eb873de-3c3a-4135-9dc0-1d441c63647c",
"name": "",
"type": "string",
"value": ""
}
]
}
},
"typeVersion": 3.3
},
{
"id": "2c7f7274-12e9-4dd3-8ee4-679b408d5430",
"name": "Fallback Response",
"type": "n8n-nodes-base.set",
"position": [
1580,
875
],
"parameters": {
"mode": "raw",
"options": {},
"jsonOutput": "{\n \"response\": {\n \"ok\": false,\n \"error\": \"The requested tool was not found or the service may be unavailable. Do not retry.\"\n }\n}\n"
},
"typeVersion": 3.3
},
{
"id": "09f36f4d-eb88-4d93-a8b3-e9ba66b46b54",
"name": "SERP Google Reverse Image API",
"type": "n8n-nodes-base.httpRequest",
"position": [
1860,
549
],
"parameters": {
"url": "https://serpapi.com/search.json",
"options": {},
"sendQuery": true,
"authentication": "predefinedCredentialType",
"queryParameters": {
"parameters": [
{
"name": "engine",
"value": "google_reverse_image"
},
{
"name": "image_url",
"value": "={{ $json.query }}"
}
]
},
"nodeCredentialType": "serpApi"
},
"credentials": {
"serpApi": {
"id": "aJCKjxx6U3K7ydDe",
"name": "SerpAPI account"
}
},
"typeVersion": 4.2
},
{
"id": "8e3a0f38-8663-4f5c-837f-4b9aa21f14fb",
"name": "Reverse Image Search Response",
"type": "n8n-nodes-base.set",
"position": [
2037,
547
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de99a504-713f-4c78-8679-08139b2def31",
"name": "response",
"type": "string",
"value": "={{ JSON.stringify($json.image_results.map(x => ({ position: x.position, title: x.title, link: x.link, description: x.snippet }))) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "0cd2269a-5b1f-4f10-b180-7f9cff9b1102",
"name": "Reverse Image Search Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
1300,
340
],
"parameters": {
"name": "reverse_image_search",
"fields": {
"values": [
{
"name": "route",
"stringValue": "serp.google_reverse_image"
}
]
},
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to perform a reverse image search. Reverse image searches return urls where similar looking products exists. Fetch the returned urls to gather more information. This tool requires the following object request body.\n```\n{\n \"type\": \"object\",\n \"properties\": {\n \"image_url\": { \"type\": \"string\" },\n }\n}\n```\nimage_url should be an absolute URL to the image."
},
"typeVersion": 1.1
},
{
"id": "9825651e-b382-4e0a-97ef-37764cb5be9e",
"name": "Firecrawl Scrape API",
"type": "n8n-nodes-base.httpRequest",
"position": [
1860,
889
],
"parameters": {
"url": "https://api.firecrawl.dev/v0/scrape",
"method": "POST",
"options": {},
"sendBody": true,
"sendHeaders": true,
"authentication": "genericCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "url",
"value": "={{ $json.query }}"
}
]
},
"genericAuthType": "httpHeaderAuth",
"headerParameters": {
"parameters": [
{
"name": "Content-Type",
"value": "application/json"
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "OUOnyTkL9vHZNorB",
"name": "Firecrawl API"
}
},
"typeVersion": 4.2
},
{
"id": "7f61d60b-b052-4b7c-abfd-9eb8e05a45a2",
"name": "Scrape Success?",
"type": "n8n-nodes-base.if",
"position": [
2020,
889
],
"parameters": {
"options": {},
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "a15a164f-d0c5-478f-8b27-f3d51746c214",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
},
"leftValue": "={{ $json.success }}",
"rightValue": ""
}
]
}
},
"typeVersion": 2
},
{
"id": "29c65ef4-6350-490a-b8e3-a5c869e656b2",
"name": "Firecrawl Scrape Success Response",
"type": "n8n-nodes-base.set",
"position": [
2180,
889
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "7db5c81f-de90-40e1-8086-3f13d40451c7",
"name": "response",
"type": "string",
"value": "={{ $json.data.markdown.substring(0, 3000) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "229b4008-d8a8-4609-854a-fc244a4ed630",
"name": "Firecrawl scrape Error Response",
"type": "n8n-nodes-base.set",
"position": [
2180,
1049
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "e691d86a-d366-44a2-baa6-3dba42527f6e",
"name": "response",
"type": "string",
"value": "{ error: \"Unable to scrape website due to unknown error. Do not retry.\" }"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "f080069b-e849-45e0-88cf-03707d22c704",
"name": "Firecrawl Web Scaper Tool",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
1440,
340
],
"parameters": {
"name": "webpage_url_scraper_tool",
"fields": {
"values": [
{
"name": "route",
"stringValue": "firecrawl.scrape"
}
]
},
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to retrieve page contents of a url.\n```\n{\n \"type\": \"object\",\n \"properties\": {\n \"url\": { \"type\": \"string\" },\n }\n}\n```\nurl should be an absolute URL."
},
"typeVersion": 1.1
},
{
"id": "4eff88bb-bd5e-4d6a-b5e1-8521632c461f",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
1500,
180
],
"parameters": {
"jsonSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"title\": { \"type\": \"string\" },\n \"description\": { \"type\": \"string\" },\n \"model\": { \"type\": \"string\" },\n \"material\": { \"type\": \"string\" },\n \"color\": { \"type\": \"string\" },\n \"condition\": { \"type\": \"string\" }\n }\n}"
},
"typeVersion": 1.1
},
{
"id": "328d106b-a473-4f54-82fd-55c30d813da9",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
280,
-260
],
"parameters": {
"color": 7,
"width": 402.5984702109446,
"height": 495.4071184783251,
"content": "## 1. Use Airtable to Capture Survey Photos\n[Read more about AirTable](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.airtable)\n\nTo enable this workflow, we need a database where we can retreive the title and photo to analyse and write the generate values back to. Airtable is perfect for this since it has a robust API we can work with.\n\nFor this demo, we'll manually trigger but this can be changed for forms or other triggers."
},
"typeVersion": 1
},
{
"id": "e358775d-ff83-411d-9364-b43c87d98134",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
716.3106363781314,
-160
],
"parameters": {
"color": 7,
"width": 359.40869874940336,
"height": 428.4787925736586,
"content": "## 2. Use AI Vision Model to Analyse the Photo.\n[Read more about OpenAI Vision](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-langchain.openai)\n\nWe'll use OpenAi vision model to create a detailed description of the product in the photo. We split this step from the agent because it uses an image model rather than the usual text-based one."
},
"typeVersion": 1
},
{
"id": "51b4a70c-9583-4e8a-8e8d-896a80ad53c3",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1111.3914848823072,
-293.9250474768817
],
"parameters": {
"color": 7,
"width": 593.0683948010671,
"height": 803.956942672397,
"content": "## 3. Build an AI Agent who Searches the Internet\n[Read more about OpenAI Agents](https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-langchain.openai)\n\nThis AI Agent has the ability to perform reverse image searches using our captured photos as well visit external webpages in order to obtain accurate product names and attributes. The Agent along with the tools might mimic what the average human user would carry out the same task.\n\n* For reverse image search, we're using SERP API service however we won't use the built-in SERP node as we need to specify custom parameters. \n* For scraping, we'll use [Firecrawl](https://www.firecrawl.dev/) as this service also helps to parse and return the page as markdown which is more efficient."
},
"typeVersion": 1
},
{
"id": "adfb519b-a5c7-432c-be32-5acfcc388b49",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1740,
-149.28190375244515
],
"parameters": {
"color": 7,
"width": 373.3601237414979,
"height": 397.7168664109706,
"content": "## 4. Overwrite our Rows with Enriched Results\n\nAnd Viola! Our AI agent has potentially saved hours of manual data entry work for our surveyor. This technique can be used for many other usecases."
},
"typeVersion": 1
},
{
"id": "6444e217-b944-450e-892a-5822d4d390ce",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
1200,
549
],
"parameters": {
"color": 7,
"width": 554.6092633638649,
"height": 490.7010880746526,
"content": "## 5. Using the Custom Workflow Tool\n[Read more about Workflow Tools](https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolworkflow)\n\nAI Agents rely on Tools to make decisions and become exponentially more powerful the more tools they have. A common pattern to manage multiple tools is to create a routing system for tools using the API pattern."
},
"typeVersion": 1
},
{
"id": "bf2459cf-a931-4232-9504-b36b15721194",
"name": "Enrich Product Rows",
"type": "n8n-nodes-base.airtable",
"position": [
1880,
60
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appbgxPBurOmQK3E7",
"cachedResultUrl": "https://airtable.com/appbgxPBurOmQK3E7",
"cachedResultName": "Building Inventory Survey Example"
},
"table": {
"__rl": true,
"mode": "id",
"value": "tblEHkoTvKpa4Aa0Q"
},
"columns": {
"value": {
"id": "={{ $('Get Applicable Rows').item.json.id }}",
"Color": "={{ $json.output.output.color }}",
"Model": "={{ $json.output.output.model }}",
"Title": "={{ $json.output.output.title }}",
"Material": "={{ $json.output.output.material }}",
"AI_status": true,
"Condition": "={{ $json.output.output.condition }}",
"Description": "={{ $json.output.output.description }}"
},
"schema": [
{
"id": "id",
"type": "string",
"display": true,
"removed": false,
"readOnly": true,
"required": false,
"displayName": "id",
"defaultMatch": true
},
{
"id": "Title",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Title",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Image",
"type": "array",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Image",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Description",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Description",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Model",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Model",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Material",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Material",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Color",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Color",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Condition",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "Condition",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "AI_status",
"type": "boolean",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "AI_status",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"id"
]
},
"options": {},
"operation": "update"
},
"credentials": {
"airtableTokenApi": {
"id": "Und0frCQ6SNVX3VV",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2
},
{
"id": "19d736bf-c29d-46a2-93bc-b536ff28c4b5",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
-100,
-260
],
"parameters": {
"width": 359.6648027457353,
"height": 381.0536322713287,
"content": "## Try It Out!\n### This workflow does the following:\n* Scans an Airtable spreadsheet for rows with product photo images.\n* Uses an AI vision model to attempt to identify the product.\n* Uses an AI Agent to research the product on the internet to enrich the product data.\n* Overwrites our Airtable spreadsheet with the enriched data.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "25f15c48-16bf-4f92-942d-c224ed88d208",
"name": "Analyse Image",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
840,
80
],
"parameters": {
"text": "=Focus on the {{ $json.Title }} in the image - we'll refer to this as the \"object\". Identify the following attributes of the object. If you cannot determine confidently, then leave blank and move to next attribute.\n* Decription of the object.\n* The model/make of the object.\n* The material(s) used in the construction of the object.\n* The color(s) of the object\n* The condition of the object. Use one of poor, good, excellent.\n",
"options": {},
"resource": "image",
"imageUrls": "={{ $json.Image[0].thumbnails.large.url }}",
"operation": "analyze"
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1.3
},
{
"id": "e6c99f71-ccc9-426e-b916-cc38864e3224",
"name": "Object Identifier Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
1260,
20
],
"parameters": {
"text": "=system: Your role is to help an building surveyor perform a object classification and data collection task whereby the surveyor will take photos of various objects and your job is to try and identify accurately certain product attributes of the objects as detailed below.\n\nThe surveyor has given you the following:\n1) photo url ```{{ $('Get Applicable Rows').item.json.Image[0].thumbnails.large.url }}```.\n2) photo description ```{{ $json.content }}```.\n\nFor each product attribute the surveyor is unable to determine, you may:\n1) use the reverse image search tool to search the product on the internet via the provided image url.\n2) use the web scraper tool to read webpages on the internet which may be relevant to the product.\n3) If after using these tools, you are still unable to determine the required product attributes then leave the data blank.\n\nUse all the information provided and gathered, to extract the following product attributes: title, description, model, material, color and condition.",
"agent": "openAiFunctionsAgent",
"options": {},
"promptType": "define",
"hasOutputParser": true
},
"typeVersion": 1.5
},
{
"id": "661b14bd-6511-4f20-981c-2e68a7c34ec5",
"name": "Actions Router",
"type": "n8n-nodes-base.switch",
"position": [
1577,
733
],
"parameters": {
"rules": {
"values": [
{
"outputKey": "serp.google_reverse_image",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"operator": {
"type": "string",
"operation": "equals"
},
"leftValue": "={{ $json.route }}",
"rightValue": "serp.google_reverse_image"
}
]
},
"renameOutput": true
},
{
"outputKey": "firecrawl.scrape",
"conditions": {
"options": {
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "0a1f54ae-39f1-468d-ba6e-1376d13e4ee8",
"operator": {
"name": "filter.operator.equals",
"type": "string",
"operation": "equals"
},
"leftValue": "={{ $json.route }}",
"rightValue": "firecrawl.scrape"
}
]
},
"renameOutput": true
}
]
},
"options": {
"fallbackOutput": "extra"
}
},
"typeVersion": 3
},
{
"id": "c5078221-9239-4ec0-b25e-7cd880b58216",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
480,
20
],
"parameters": {
"width": 181.2788838920522,
"height": 297.0159375852115,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set Airtable Base and Table IDs here."
},
"typeVersion": 1
},
{
"id": "c58c0db4-9b99-4a77-90ae-66fa3981b684",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1840,
40
],
"parameters": {
"width": 181.2788838920522,
"height": 297.0159375852115,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n🚨**Required**\n* Set Airtable Base and Table IDs here."
},
"typeVersion": 1
},
{
"id": "e3a666d7-d7a5-43f5-8f04-7972332f8916",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
1780,
440
],
"parameters": {
"color": 7,
"width": 460.3301604548244,
"height": 298.81538450684064,
"content": "## 5.1 Google Reverse Image Tool\nThis tool uses Google's reverse image API to return websites where similar images are found."
},
"typeVersion": 1
},
{
"id": "d7407cdb-16bb-4bd9-a28e-7a72a5289354",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
1780,
769.9385328672522
],
"parameters": {
"color": 7,
"width": 575.3216480295998,
"height": 463.34699288922565,
"content": "## 5.2 Webscraper Tool\nThis tool uses Firecrawl.dev API to crawl webpages and returns those pages in markdown format."
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Edit Fields": {
"main": [
[
{
"node": "Actions Router",
"type": "main",
"index": 0
}
]
]
},
"Analyse Image": {
"main": [
[
{
"node": "Object Identifier Agent",
"type": "main",
"index": 0
}
]
]
},
"Actions Router": {
"main": [
[
{
"node": "SERP Google Reverse Image API",
"type": "main",
"index": 0
}
],
[
{
"node": "Firecrawl Scrape API",
"type": "main",
"index": 0
}
],
[
{
"node": "Fallback Response",
"type": "main",
"index": 0
}
]
]
},
"Scrape Success?": {
"main": [
[
{
"node": "Firecrawl Scrape Success Response",
"type": "main",
"index": 0
}
],
[
{
"node": "Firecrawl scrape Error Response",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Object Identifier Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Get Applicable Rows": {
"main": [
[
{
"node": "Analyse Image",
"type": "main",
"index": 0
}
]
]
},
"Firecrawl Scrape API": {
"main": [
[
{
"node": "Scrape Success?",
"type": "main",
"index": 0
}
]
]
},
"Object Identifier Agent": {
"main": [
[
{
"node": "Enrich Product Rows",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Edit Fields",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Object Identifier Agent",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Firecrawl Web Scaper Tool": {
"ai_tool": [
[
{
"node": "Object Identifier Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Reverse Image Search Tool": {
"ai_tool": [
[
{
"node": "Object Identifier Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"SERP Google Reverse Image API": {
"main": [
[
{
"node": "Reverse Image Search Response",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Get Applicable Rows",
"type": "main",
"index": 0
}
]
]
}
}
}

1194
AI_Research_RAG_and_Data_Analysis/Extract insights & analyse YouTube comments via AI Agent chat.json Normal file
View File

File diff suppressed because it is too large Load Diff

1194
AI_Research_RAG_and_Data_Analysis/Extract insights & analyse YouTube comments via AI Agent chat.txt
View File

File diff suppressed because it is too large Load Diff

338
AI_Research_RAG_and_Data_Analysis/Generate SEO Seed Keywords Using AI.json Normal file
View File

@@ -0,0 +1,338 @@
{
"meta": {
"instanceId": "257476b1ef58bf3cb6a46e65fac7ee34a53a5e1a8492d5c6e4da5f87c9b82833",
"templateId": "2473"
},
"nodes": [
{
"id": "1205b121-8aaa-4e41-874b-4e81aad6374e",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
600
],
"parameters": {
"color": 4,
"width": 462.4041757955455,
"height": 315.6388466176832,
"content": "## Generate SEO Seed Keywords Using AI\n\nThis flow uses an AI node to generate Seed Keywords to focus SEO efforts on based on your ideal customer profile\n\n**Outputs:** \n- List of 20 Seed Keywords\n\n\n**Pre-requisites / Dependencies:**\n- You know your ideal customer profile (ICP)\n- An AI API account (either OpenAI or Anthropic recommended)"
},
"typeVersion": 1
},
{
"id": "d2654d75-2b64-4ec3-b583-57d2b6b7b195",
"name": "Sticky Note13",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
640,
920
],
"parameters": {
"color": 7,
"width": 287.0816455493243,
"height": 330.47923074942287,
"content": "**Generate draft seed KW based on ICP**\n\n"
},
"typeVersion": 1
},
{
"id": "d248a58e-3705-4b6f-99cb-e9187e56781c",
"name": "Anthropic Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatAnthropic",
"position": [
680,
1120
],
"parameters": {
"options": {}
},
"typeVersion": 1.2
},
{
"id": "71517d83-59f5-441a-8a75-c35f4e06a8a2",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
980,
980
],
"parameters": {
"options": {},
"fieldToSplitOut": "output.answer"
},
"typeVersion": 1
},
{
"id": "1c68eff5-6478-4eba-9abe-3ccea2a17a5c",
"name": "Sticky Note12",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
120,
920
],
"parameters": {
"color": 7,
"width": 492.16246201447336,
"height": 213.62075341687063,
"content": "**Get data from airtable and format** "
},
"typeVersion": 1
},
{
"id": "53dcc524-ef3d-40b8-b79d-976517dce4e7",
"name": "Sticky Note17",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
960,
920
],
"parameters": {
"color": 7,
"width": 348.42891651921957,
"height": 213.62075341687063,
"content": "**Add data to database**"
},
"typeVersion": 1
},
{
"id": "570495fe-3f1d-44ae-bea0-9fa4b2ce15ef",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
640,
820
],
"parameters": {
"color": 6,
"width": 393.46745700785266,
"height": 80,
"content": "**Costs to run**\nApprox. $0.02-0.05 for a run using Claude Sonnet 3.5"
},
"typeVersion": 1
},
{
"id": "6e5e84c5-409f-4f37-931a-21a44aff7c5e",
"name": "Set Ideal Customer Profile (ICP)",
"type": "n8n-nodes-base.set",
"position": [
160,
980
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "973e949e-1afd-4378-8482-d2168532eff6",
"name": "product",
"type": "string",
"value": "=**Replace this with a string detailing your intended product (if you have one)**"
},
{
"id": "ce9c0a8f-6157-4b46-8b77-133545dc71bd",
"name": "pain points",
"type": "string",
"value": "=**Replace this with a string list of customer pain points**"
},
{
"id": "5abc858a-c412-4acf-acb9-488e4d992d2f",
"name": "goals",
"type": "string",
"value": "=**Replace this with a string list of your customers key goals/objectives**"
},
{
"id": "fbdd1ef7-c1b9-48eb-b73e-a383f12b5ba1",
"name": "current solutions",
"type": "string",
"value": "=**Replace this with a string detailing how your ideal customer currently solves their pain ppoints**"
},
{
"id": "2e5c8f48-266e-486c-956f-51f1449f6288",
"name": "expertise level",
"type": "string",
"value": "=**Replace this with a string detailing customer level of expertise**"
}
]
}
},
"notesInFlow": true,
"typeVersion": 3.4
},
{
"id": "bd5781f4-6f35-45d3-8182-12ea6712eddf",
"name": "Aggregate for AI node",
"type": "n8n-nodes-base.aggregate",
"position": [
380,
980
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData"
},
"notesInFlow": true,
"typeVersion": 1
},
{
"id": "244943bf-e4dd-40fc-9a43-7a5cd0da1c5b",
"name": "Sticky Note14",
"type": "n8n-nodes-base.stickyNote",
"position": [
640,
1260
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nConnect to your own AI API above"
},
"typeVersion": 1
},
{
"id": "73c8f47a-4fdb-40c8-9062-890ef1265ab0",
"name": "Sticky Note16",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
1140
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nSet your Ideal Customer Profile before proceeding"
},
"typeVersion": 1
},
{
"id": "a5b93e6d-44ab-4b6f-b86a-25dc621b52b0",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
660,
980
],
"parameters": {
"text": "=User:\nHere are some important rules for you to follow:\n<rules>\n1. Analyze the ICP information carefully.\n2. Generate 15-20 seed keywords that are relevant to the ICP's needs, challenges, goals, and search behavior.\n3. Ensure the keywords are broad enough to be considered \"\"head\"\" terms, but specific enough to target the ICP effectively.\n4. Consider various aspects of the ICP's journey, including awareness, consideration, and decision stages.\n5. Include a mix of product-related, problem-related, and solution-related terms.\n6. Think beyond just the product itself - consider industry trends, related technologies, and broader business concepts that would interest the ICP.\n7. Avoid overly generic terms that might attract irrelevant traffic.\n8. Aim for a mix of keyword difficulties, including both competitive and less competitive terms.\n9. Include keywords that cover different search intents: informational, navigational, commercial, and transactional.\n10. Consider related tools or platforms that the ICP might use, and include relevant integration-related keywords.\n11. If applicable, include some location-specific keywords based on the ICP's geographic information.\n12. Incorporate industry-specific terminology or jargon that the ICP would likely use in their searches.\n13. Consider emerging trends or pain points in the ICP's industry that they might be searching for solutions to.\n13. Format the keywords in lowercase, without punctuation. Trim any leading or trailing white space.\n</rules>\n\nYour output should be an array of strings, each representing a seed keyword:\n<example>\n['b2b lead generation', 'startup marketing strategies', 'saas sales funnel', ...]\n</example>\n\nHere is the Ideal Customer Profile (ICP) information:\n<input>\n{{ $json.data[0].product }}\n</input>\n\nNow:\nBased on the provided ICP, generate an array of 15-20 seed keywords that will form the foundation of a comprehensive SEO strategy for this B2B SaaS company. These keywords should reflect a deep understanding of the ICP's needs, challenges, and search behavior, while also considering broader industry trends and related concepts.\n\nFirst, write out your ideas in {thoughts: } JSON as part of your analysis, then answer inside the {answer: } key in the JSON. ",
"agent": "conversationalAgent",
"options": {
"systemMessage": "=System: You are an expert SEO strategist tasked with generating 15-20 key head search terms (seed keywords) for a B2B SaaS company. Your goal is to create a comprehensive list of keywords that will attract and engage the ideal customer profile (ICP) described."
},
"promptType": "define"
},
"typeVersion": 1.6
},
{
"id": "ca3c0bd5-7ef0-4e2b-9b5e-071773c32c85",
"name": "Connect to your own database",
"type": "n8n-nodes-base.noOp",
"position": [
1140,
980
],
"parameters": {},
"typeVersion": 1
},
{
"id": "94639a81-5e46-482a-851a-5443bfe9863c",
"name": "Sticky Note15",
"type": "n8n-nodes-base.stickyNote",
"position": [
1120,
1140
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nConnect to your own database / GSheet / Airtable base to output these"
},
"typeVersion": 1
},
{
"id": "16498e92-c0d5-44f4-b993-c9c8930955bc",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-60,
980
],
"parameters": {},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"AI Agent": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Connect to your own database",
"type": "main",
"index": 0
}
]
]
},
"Anthropic Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Aggregate for AI node": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Set Ideal Customer Profile (ICP)": {
"main": [
[
{
"node": "Aggregate for AI node",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Ideal Customer Profile (ICP)",
"type": "main",
"index": 0
}
]
]
}
}
}

338
AI_Research_RAG_and_Data_Analysis/Generate SEO Seed Keywords Using AI.txt
View File

@@ -1,338 +0,0 @@
{
"meta": {
"instanceId": "257476b1ef58bf3cb6a46e65fac7ee34a53a5e1a8492d5c6e4da5f87c9b82833",
"templateId": "2473"
},
"nodes": [
{
"id": "1205b121-8aaa-4e41-874b-4e81aad6374e",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
600
],
"parameters": {
"color": 4,
"width": 462.4041757955455,
"height": 315.6388466176832,
"content": "## Generate SEO Seed Keywords Using AI\n\nThis flow uses an AI node to generate Seed Keywords to focus SEO efforts on based on your ideal customer profile\n\n**Outputs:** \n- List of 20 Seed Keywords\n\n\n**Pre-requisites / Dependencies:**\n- You know your ideal customer profile (ICP)\n- An AI API account (either OpenAI or Anthropic recommended)"
},
"typeVersion": 1
},
{
"id": "d2654d75-2b64-4ec3-b583-57d2b6b7b195",
"name": "Sticky Note13",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
640,
920
],
"parameters": {
"color": 7,
"width": 287.0816455493243,
"height": 330.47923074942287,
"content": "**Generate draft seed KW based on ICP**\n\n"
},
"typeVersion": 1
},
{
"id": "d248a58e-3705-4b6f-99cb-e9187e56781c",
"name": "Anthropic Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatAnthropic",
"position": [
680,
1120
],
"parameters": {
"options": {}
},
"typeVersion": 1.2
},
{
"id": "71517d83-59f5-441a-8a75-c35f4e06a8a2",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
980,
980
],
"parameters": {
"options": {},
"fieldToSplitOut": "output.answer"
},
"typeVersion": 1
},
{
"id": "1c68eff5-6478-4eba-9abe-3ccea2a17a5c",
"name": "Sticky Note12",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
120,
920
],
"parameters": {
"color": 7,
"width": 492.16246201447336,
"height": 213.62075341687063,
"content": "**Get data from airtable and format** "
},
"typeVersion": 1
},
{
"id": "53dcc524-ef3d-40b8-b79d-976517dce4e7",
"name": "Sticky Note17",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
960,
920
],
"parameters": {
"color": 7,
"width": 348.42891651921957,
"height": 213.62075341687063,
"content": "**Add data to database**"
},
"typeVersion": 1
},
{
"id": "570495fe-3f1d-44ae-bea0-9fa4b2ce15ef",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
640,
820
],
"parameters": {
"color": 6,
"width": 393.46745700785266,
"height": 80,
"content": "**Costs to run**\nApprox. $0.02-0.05 for a run using Claude Sonnet 3.5"
},
"typeVersion": 1
},
{
"id": "6e5e84c5-409f-4f37-931a-21a44aff7c5e",
"name": "Set Ideal Customer Profile (ICP)",
"type": "n8n-nodes-base.set",
"position": [
160,
980
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "973e949e-1afd-4378-8482-d2168532eff6",
"name": "product",
"type": "string",
"value": "=**Replace this with a string detailing your intended product (if you have one)**"
},
{
"id": "ce9c0a8f-6157-4b46-8b77-133545dc71bd",
"name": "pain points",
"type": "string",
"value": "=**Replace this with a string list of customer pain points**"
},
{
"id": "5abc858a-c412-4acf-acb9-488e4d992d2f",
"name": "goals",
"type": "string",
"value": "=**Replace this with a string list of your customers key goals/objectives**"
},
{
"id": "fbdd1ef7-c1b9-48eb-b73e-a383f12b5ba1",
"name": "current solutions",
"type": "string",
"value": "=**Replace this with a string detailing how your ideal customer currently solves their pain ppoints**"
},
{
"id": "2e5c8f48-266e-486c-956f-51f1449f6288",
"name": "expertise level",
"type": "string",
"value": "=**Replace this with a string detailing customer level of expertise**"
}
]
}
},
"notesInFlow": true,
"typeVersion": 3.4
},
{
"id": "bd5781f4-6f35-45d3-8182-12ea6712eddf",
"name": "Aggregate for AI node",
"type": "n8n-nodes-base.aggregate",
"position": [
380,
980
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData"
},
"notesInFlow": true,
"typeVersion": 1
},
{
"id": "244943bf-e4dd-40fc-9a43-7a5cd0da1c5b",
"name": "Sticky Note14",
"type": "n8n-nodes-base.stickyNote",
"position": [
640,
1260
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nConnect to your own AI API above"
},
"typeVersion": 1
},
{
"id": "73c8f47a-4fdb-40c8-9062-890ef1265ab0",
"name": "Sticky Note16",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
1140
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nSet your Ideal Customer Profile before proceeding"
},
"typeVersion": 1
},
{
"id": "a5b93e6d-44ab-4b6f-b86a-25dc621b52b0",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
660,
980
],
"parameters": {
"text": "=User:\nHere are some important rules for you to follow:\n<rules>\n1. Analyze the ICP information carefully.\n2. Generate 15-20 seed keywords that are relevant to the ICP's needs, challenges, goals, and search behavior.\n3. Ensure the keywords are broad enough to be considered \"\"head\"\" terms, but specific enough to target the ICP effectively.\n4. Consider various aspects of the ICP's journey, including awareness, consideration, and decision stages.\n5. Include a mix of product-related, problem-related, and solution-related terms.\n6. Think beyond just the product itself - consider industry trends, related technologies, and broader business concepts that would interest the ICP.\n7. Avoid overly generic terms that might attract irrelevant traffic.\n8. Aim for a mix of keyword difficulties, including both competitive and less competitive terms.\n9. Include keywords that cover different search intents: informational, navigational, commercial, and transactional.\n10. Consider related tools or platforms that the ICP might use, and include relevant integration-related keywords.\n11. If applicable, include some location-specific keywords based on the ICP's geographic information.\n12. Incorporate industry-specific terminology or jargon that the ICP would likely use in their searches.\n13. Consider emerging trends or pain points in the ICP's industry that they might be searching for solutions to.\n13. Format the keywords in lowercase, without punctuation. Trim any leading or trailing white space.\n</rules>\n\nYour output should be an array of strings, each representing a seed keyword:\n<example>\n['b2b lead generation', 'startup marketing strategies', 'saas sales funnel', ...]\n</example>\n\nHere is the Ideal Customer Profile (ICP) information:\n<input>\n{{ $json.data[0].product }}\n</input>\n\nNow:\nBased on the provided ICP, generate an array of 15-20 seed keywords that will form the foundation of a comprehensive SEO strategy for this B2B SaaS company. These keywords should reflect a deep understanding of the ICP's needs, challenges, and search behavior, while also considering broader industry trends and related concepts.\n\nFirst, write out your ideas in {thoughts: } JSON as part of your analysis, then answer inside the {answer: } key in the JSON. ",
"agent": "conversationalAgent",
"options": {
"systemMessage": "=System: You are an expert SEO strategist tasked with generating 15-20 key head search terms (seed keywords) for a B2B SaaS company. Your goal is to create a comprehensive list of keywords that will attract and engage the ideal customer profile (ICP) described."
},
"promptType": "define"
},
"typeVersion": 1.6
},
{
"id": "ca3c0bd5-7ef0-4e2b-9b5e-071773c32c85",
"name": "Connect to your own database",
"type": "n8n-nodes-base.noOp",
"position": [
1140,
980
],
"parameters": {},
"typeVersion": 1
},
{
"id": "94639a81-5e46-482a-851a-5443bfe9863c",
"name": "Sticky Note15",
"type": "n8n-nodes-base.stickyNote",
"position": [
1120,
1140
],
"parameters": {
"color": 3,
"width": 284.87764467541297,
"height": 80,
"content": "**REQUIRED**\nConnect to your own database / GSheet / Airtable base to output these"
},
"typeVersion": 1
},
{
"id": "16498e92-c0d5-44f4-b993-c9c8930955bc",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-60,
980
],
"parameters": {},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"AI Agent": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Connect to your own database",
"type": "main",
"index": 0
}
]
]
},
"Anthropic Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Aggregate for AI node": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Set Ideal Customer Profile (ICP)": {
"main": [
[
{
"node": "Aggregate for AI node",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Ideal Customer Profile (ICP)",
"type": "main",
"index": 0
}
]
]
}
}
}

745
AI_Research_RAG_and_Data_Analysis/Hacker News Job Listing Scraper and Parser.json Normal file
View File

@@ -0,0 +1,745 @@
{
"id": "0JsHmmyeHw5Ffz5m",
"meta": {
"instanceId": "d4d7965840e96e50a3e02959a8487c692901dfa8d5cc294134442c67ce1622d3",
"templateCredsSetupCompleted": true
},
"name": "HN Who is Hiring Scrape",
"tags": [],
"nodes": [
{
"id": "f7cdb3ee-9bb0-4006-829a-d4ce797191d5",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-20,
-220
],
"parameters": {},
"typeVersion": 1
},
{
"id": "0475e25d-9bf4-450d-abd3-a04608a438a4",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
60,
-620
],
"parameters": {
"width": 460,
"height": 340,
"content": "## Go to https://hn.algolia.com\n- filter by \"Ask HN: Who is hiring?\" (important with quotes for full match)\n- sort by date\n- Chrome Network Tab > find API call > click \"Copy as cURL\"\n- n8n HTTP node -> import cURL and paste \n- I've set the API key as Header Auth so you will have to do the above yourself to make this work"
},
"typeVersion": 1
},
{
"id": "a686852b-ff84-430b-92bb-ce02a6808e19",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
400,
-220
],
"parameters": {
"options": {},
"fieldToSplitOut": "hits"
},
"typeVersion": 1
},
{
"id": "cdaaa738-d561-4fa0-b2c7-8ea9e6778eb1",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1260,
-620
],
"parameters": {
"width": 500,
"height": 340,
"content": "## Go to HN API \nhttps://github.com/HackerNews/API\n\nWe'll need following endpoints: \n- For example, a story: https://hacker-news.firebaseio.com/v0/item/8863.json?print=pretty\n- comment: https://hacker-news.firebaseio.com/v0/item/2921983.json?print=pretty\n\n"
},
"typeVersion": 1
},
{
"id": "4f353598-9e32-4be4-9e7b-c89cc05305fd",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
2680,
-20
],
"parameters": {
"model": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini"
},
"options": {}
},
"credentials": {
"openAiApi": {
"id": "Fbb2ueT0XP5xMRme",
"name": "OpenAi account 2"
}
},
"typeVersion": 1.2
},
{
"id": "5bd0d7cc-497a-497c-aa4c-589d9ceeca14",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
2840,
-20
],
"parameters": {
"schemaType": "manual",
"inputSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"company\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Name of the hiring company\"\n },\n \"title\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Job title/role being advertised\"\n },\n \"location\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Work location including remote/hybrid status\"\n },\n \"type\": {\n \"type\": [\n \"string\",\n null\n ],\n \"enum\": [\n \"FULL_TIME\",\n \"PART_TIME\",\n \"CONTRACT\",\n \"INTERNSHIP\",\n \"FREELANCE\",\n null\n ],\n \"description\": \"Employment type (Full-time, Contract, etc)\"\n },\n \"work_location\": {\n \"type\": [\n \"string\",\n null\n ],\n \"enum\": [\n \"REMOTE\",\n \"HYBRID\",\n \"ON_SITE\",\n null\n ],\n \"description\": \"Work arrangement type\"\n },\n \"salary\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Compensation details if provided\"\n },\n \"description\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Main job description text including requirements and team info\"\n },\n \"apply_url\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Direct application/job posting URL\"\n },\n \"company_url\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Company website or careers page\"\n }\n }\n}\n"
},
"typeVersion": 1.2
},
{
"id": "b84ca004-6f3b-4577-8910-61b8584b161d",
"name": "Search for Who is hiring posts",
"type": "n8n-nodes-base.httpRequest",
"position": [
200,
-220
],
"parameters": {
"url": "https://uj5wyc0l7x-dsn.algolia.net/1/indexes/Item_dev_sort_date/query",
"method": "POST",
"options": {},
"jsonBody": "{\n \"query\": \"\\\"Ask HN: Who is hiring\\\"\",\n \"analyticsTags\": [\n \"web\"\n ],\n \"page\": 0,\n \"hitsPerPage\": 30,\n \"minWordSizefor1Typo\": 4,\n \"minWordSizefor2Typos\": 8,\n \"advancedSyntax\": true,\n \"ignorePlurals\": false,\n \"clickAnalytics\": true,\n \"minProximity\": 7,\n \"numericFilters\": [],\n \"tagFilters\": [\n [\n \"story\"\n ],\n []\n ],\n \"typoTolerance\": \"min\",\n \"queryType\": \"prefixNone\",\n \"restrictSearchableAttributes\": [\n \"title\",\n \"comment_text\",\n \"url\",\n \"story_text\",\n \"author\"\n ],\n \"getRankingInfo\": true\n}",
"sendBody": true,
"sendQuery": true,
"sendHeaders": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"queryParameters": {
"parameters": [
{
"name": "x-algolia-agent",
"value": "Algolia for JavaScript (4.13.1); Browser (lite)"
},
{
"name": "x-algolia-application-id",
"value": "UJ5WYC0L7X"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Accept",
"value": "*/*"
},
{
"name": "Accept-Language",
"value": "en-GB,en-US;q=0.9,en;q=0.8"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "DNT",
"value": "1"
},
{
"name": "Origin",
"value": "https://hn.algolia.com"
},
{
"name": "Referer",
"value": "https://hn.algolia.com/"
},
{
"name": "Sec-Fetch-Dest",
"value": "empty"
},
{
"name": "Sec-Fetch-Mode",
"value": "cors"
},
{
"name": "Sec-Fetch-Site",
"value": "cross-site"
},
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
},
{
"name": "sec-ch-ua",
"value": "\"Chromium\";v=\"133\", \"Not(A:Brand\";v=\"99\""
},
{
"name": "sec-ch-ua-mobile",
"value": "?0"
},
{
"name": "sec-ch-ua-platform",
"value": "\"macOS\""
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "oVEXp2ZbYCXypMVz",
"name": "Algolia Auth"
}
},
"typeVersion": 4.2
},
{
"id": "205e66f6-cd6b-4cfd-a6ec-2226c35ddaac",
"name": "Get relevant data",
"type": "n8n-nodes-base.set",
"position": [
700,
-220
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "73dd2325-faa7-4650-bd78-5fc97cc202de",
"name": "title",
"type": "string",
"value": "={{ $json.title }}"
},
{
"id": "44918eac-4510-440e-9ac0-bf14d2b2f3af",
"name": "createdAt",
"type": "string",
"value": "={{ $json.created_at }}"
},
{
"id": "00eb6f09-2c22-411c-949c-886b2d95b6eb",
"name": "updatedAt",
"type": "string",
"value": "={{ $json.updated_at }}"
},
{
"id": "2b4f9da6-f60e-46e0-ba9d-3242fa955a55",
"name": "storyId",
"type": "string",
"value": "={{ $json.story_id }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "16bc5628-8a29-4eac-8be9-b4e9da802e1e",
"name": "Get latest post",
"type": "n8n-nodes-base.filter",
"position": [
900,
-220
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "d7dd7175-2a50-45aa-bd3e-4c248c9193c4",
"operator": {
"type": "dateTime",
"operation": "after"
},
"leftValue": "={{ $json.createdAt }}",
"rightValue": "={{$now.minus({days: 30})}} "
}
]
}
},
"typeVersion": 2.2
},
{
"id": "92e1ef74-5ae1-4195-840b-115184db464f",
"name": "Split out children (jobs)",
"type": "n8n-nodes-base.splitOut",
"position": [
1460,
-220
],
"parameters": {
"options": {},
"fieldToSplitOut": "kids"
},
"typeVersion": 1
},
{
"id": "d0836aae-b98a-497f-a6f7-0ad563c262a0",
"name": "Trun into structured data",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
2600,
-220
],
"parameters": {
"text": "={{ $json.cleaned_text }}",
"messages": {
"messageValues": [
{
"message": "Extract the JSON data"
}
]
},
"promptType": "define",
"hasOutputParser": true
},
"typeVersion": 1.5
},
{
"id": "fd818a93-627c-435d-91ba-5d759d5a9004",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2600,
-620
],
"parameters": {
"width": 840,
"height": 340,
"content": "## Data Structure\n\nWe use Openai GPT-4o-mini to transform the raw data in a unified data structure. Feel free to change this.\n\n```json\n{\n \"company\": \"Name of the hiring company\",\n \"title\": \"Job title/role being advertised\",\n \"location\": \"Work location including remote/hybrid status\",\n \"type\": \"Employment type (Full-time, Contract, etc)\",\n \"salary\": \"Compensation details if provided\",\n \"description\": \"Main job description text including requirements and team info\",\n \"apply_url\": \"Direct application/job posting URL\",\n \"company_url\": \"Company website or careers page\"\n}\n```"
},
"typeVersion": 1
},
{
"id": "b70c5578-5b81-467a-8ac2-65374e4e52f3",
"name": "Extract text",
"type": "n8n-nodes-base.set",
"position": [
1860,
-220
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "6affa370-56ce-4ad8-8534-8f753fdf07fc",
"name": "text",
"type": "string",
"value": "={{ $json.text }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "acb68d88-9417-42e9-9bcc-7c2fa95c4afd",
"name": "Clean text",
"type": "n8n-nodes-base.code",
"position": [
2060,
-220
],
"parameters": {
"jsCode": "// In a Function node in n8n\nconst inputData = $input.all();\n\nfunction cleanAllPosts(data) {\n return data.map(item => {\n try {\n // Check if item exists and has the expected structure\n if (!item || typeof item !== 'object') {\n return { cleaned_text: '', error: 'Invalid item structure' };\n }\n\n // Get the text, with multiple fallbacks\n let text = '';\n if (typeof item === 'string') {\n text = item;\n } else if (item.json && item.json.text) {\n text = item.json.text;\n } else if (typeof item.json === 'string') {\n text = item.json;\n } else {\n text = JSON.stringify(item);\n }\n\n // Make sure text is a string\n text = String(text);\n \n // Perform the cleaning operations\n try {\n text = text.replace(/&#x2F;/g, '/');\n text = text.replace(/&#x27;/g, \"'\");\n text = text.replace(/&\\w+;/g, ' ');\n text = text.replace(/<[^>]*>/g, '');\n text = text.replace(/\\|\\s*/g, '| ');\n text = text.replace(/\\s+/g, ' ');\n text = text.replace(/\\s*(https?:\\/\\/[^\\s]+)\\s*/g, '\\n$1\\n');\n text = text.replace(/\\n{3,}/g, '\\n\\n');\n text = text.trim();\n } catch (cleaningError) {\n console.log('Error during text cleaning:', cleaningError);\n // Return original text if cleaning fails\n return { cleaned_text: text, warning: 'Partial cleaning applied' };\n }\n\n return { cleaned_text: text };\n \n } catch (error) {\n console.log('Error processing item:', error);\n return { \n cleaned_text: '', \n error: `Processing error: ${error.message}`,\n original: item\n };\n }\n }).filter(result => result.cleaned_text || result.error); \n}\n\ntry {\n return cleanAllPosts(inputData);\n} catch (error) {\n console.log('Fatal error:', error);\n return [{ \n cleaned_text: '', \n error: `Fatal error: ${error.message}`,\n input: inputData \n }];\n}\n"
},
"typeVersion": 2
},
{
"id": "a0727b55-565d-47c0-9ab5-0f001f4b9941",
"name": "Limit for testing (optional)",
"type": "n8n-nodes-base.limit",
"position": [
2280,
-220
],
"parameters": {
"maxItems": 5
},
"typeVersion": 1
},
{
"id": "650baf5e-c2ac-443d-8a2b-6df89717186f",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
580,
-620
],
"parameters": {
"width": 540,
"height": 340,
"content": "## Clean the result \n\n```json\n{\n\"title\": \"Ask HN: Who is hiring? (February 2025)\",\n\"createdAt\": \"2025-02-03T16:00:43Z\",\n\"updatedAt\": \"2025-02-17T08:35:44Z\",\n\"storyId\": \"42919502\"\n},\n{\n\"title\": \"Ask HN: Who is hiring? (January 2025)\",\n\"createdAt\": \"2025-01-02T16:00:09Z\",\n\"updatedAt\": \"2025-02-13T00:03:24Z\",\n\"storyId\": \"42575537\"\n},\n```"
},
"typeVersion": 1
},
{
"id": "1ca5c39f-f21d-455a-b63a-702e7e3ba02b",
"name": "Write results to airtable",
"type": "n8n-nodes-base.airtable",
"position": [
3040,
-220
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appM2JWvA5AstsGdn",
"cachedResultUrl": "https://airtable.com/appM2JWvA5AstsGdn",
"cachedResultName": "HN Who is hiring?"
},
"table": {
"__rl": true,
"mode": "list",
"value": "tblGvcOjqbliwM7AS",
"cachedResultUrl": "https://airtable.com/appM2JWvA5AstsGdn/tblGvcOjqbliwM7AS",
"cachedResultName": "Table 1"
},
"columns": {
"value": {
"type": "={{ $json.output.type }}",
"title": "={{ $json.output.title }}",
"salary": "={{ $json.output.salary }}",
"company": "={{ $json.output.company }}",
"location": "={{ $json.output.location }}",
"apply_url": "={{ $json.output.apply_url }}",
"company_url": "={{ $json.output.company_url }}",
"description": "={{ $json.output.description }}"
},
"schema": [
{
"id": "title",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "title",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "company",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "company",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "location",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "location",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "type",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "type",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "salary",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "salary",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "description",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "description",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "apply_url",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "apply_url",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "company_url",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "company_url",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "posted_date",
"type": "string",
"display": true,
"removed": true,
"readOnly": false,
"required": false,
"displayName": "posted_date",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "create"
},
"credentials": {
"airtableTokenApi": {
"id": "IudXLNj7CDuc5M5a",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2.1
},
{
"id": "d71fa024-86a0-4f74-b033-1f755574080c",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-300
],
"parameters": {
"width": 380,
"height": 500,
"content": "## Hacker News - Who is Hiring Scrape\n\nIn this template we setup a scraper for the monthly HN Who is Hiring post. This way we can scrape the data and transform it to a common data strcutre.\n\nFirst we use the [Algolia Search](https://hn.algolia.com/) provided by hackernews to drill down the results.\n\nWe can use the official [Hacker News API](https://github.com/HackerNews/API\n) to get the post data and also all the replies!\n\nThis will obviously work for any kind of post on hacker news! Get creative 😃\n\nAll you need is an Openai Account to structure the text data and an Airtable Account (or similar) to write the results to a list.\n\nCopy my base https://airtable.com/appM2JWvA5AstsGdn/shrAuo78cJt5C2laR"
},
"typeVersion": 1
},
{
"id": "7466fb0c-9f0c-4adf-a6de-b2cf09032719",
"name": "HI API: Get the individual job post",
"type": "n8n-nodes-base.httpRequest",
"position": [
1660,
-220
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.kids }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "184abccf-5838-49bf-9922-e0300c6b145e",
"name": "HN API: Get Main Post",
"type": "n8n-nodes-base.httpRequest",
"position": [
1260,
-220
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.storyId }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "387f7084-58fa-4643-9351-73c870d3f028",
"connections": {
"Split Out": {
"main": [
[
{
"node": "Get relevant data",
"type": "main",
"index": 0
}
]
]
},
"Clean text": {
"main": [
[
{
"node": "Limit for testing (optional)",
"type": "main",
"index": 0
}
]
]
},
"Extract text": {
"main": [
[
{
"node": "Clean text",
"type": "main",
"index": 0
}
]
]
},
"Get latest post": {
"main": [
[
{
"node": "HN API: Get Main Post",
"type": "main",
"index": 0
}
]
]
},
"Get relevant data": {
"main": [
[
{
"node": "Get latest post",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Trun into structured data",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"HN API: Get Main Post": {
"main": [
[
{
"node": "Split out children (jobs)",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Trun into structured data",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Split out children (jobs)": {
"main": [
[
{
"node": "HI API: Get the individual job post",
"type": "main",
"index": 0
}
]
]
},
"Trun into structured data": {
"main": [
[
{
"node": "Write results to airtable",
"type": "main",
"index": 0
}
]
]
},
"Limit for testing (optional)": {
"main": [
[
{
"node": "Trun into structured data",
"type": "main",
"index": 0
}
]
]
},
"Search for Who is hiring posts": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Search for Who is hiring posts",
"type": "main",
"index": 0
}
]
]
},
"HI API: Get the individual job post": {
"main": [
[
{
"node": "Extract text",
"type": "main",
"index": 0
}
]
]
}
}
}

745
AI_Research_RAG_and_Data_Analysis/Hacker News Job Listing Scraper and Parser.txt
View File

@@ -1,745 +0,0 @@
{
"id": "0JsHmmyeHw5Ffz5m",
"meta": {
"instanceId": "d4d7965840e96e50a3e02959a8487c692901dfa8d5cc294134442c67ce1622d3",
"templateCredsSetupCompleted": true
},
"name": "HN Who is Hiring Scrape",
"tags": [],
"nodes": [
{
"id": "f7cdb3ee-9bb0-4006-829a-d4ce797191d5",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-20,
-220
],
"parameters": {},
"typeVersion": 1
},
{
"id": "0475e25d-9bf4-450d-abd3-a04608a438a4",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
60,
-620
],
"parameters": {
"width": 460,
"height": 340,
"content": "## Go to https://hn.algolia.com\n- filter by \"Ask HN: Who is hiring?\" (important with quotes for full match)\n- sort by date\n- Chrome Network Tab > find API call > click \"Copy as cURL\"\n- n8n HTTP node -> import cURL and paste \n- I've set the API key as Header Auth so you will have to do the above yourself to make this work"
},
"typeVersion": 1
},
{
"id": "a686852b-ff84-430b-92bb-ce02a6808e19",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
400,
-220
],
"parameters": {
"options": {},
"fieldToSplitOut": "hits"
},
"typeVersion": 1
},
{
"id": "cdaaa738-d561-4fa0-b2c7-8ea9e6778eb1",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1260,
-620
],
"parameters": {
"width": 500,
"height": 340,
"content": "## Go to HN API \nhttps://github.com/HackerNews/API\n\nWe'll need following endpoints: \n- For example, a story: https://hacker-news.firebaseio.com/v0/item/8863.json?print=pretty\n- comment: https://hacker-news.firebaseio.com/v0/item/2921983.json?print=pretty\n\n"
},
"typeVersion": 1
},
{
"id": "4f353598-9e32-4be4-9e7b-c89cc05305fd",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
2680,
-20
],
"parameters": {
"model": {
"__rl": true,
"mode": "list",
"value": "gpt-4o-mini"
},
"options": {}
},
"credentials": {
"openAiApi": {
"id": "Fbb2ueT0XP5xMRme",
"name": "OpenAi account 2"
}
},
"typeVersion": 1.2
},
{
"id": "5bd0d7cc-497a-497c-aa4c-589d9ceeca14",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
2840,
-20
],
"parameters": {
"schemaType": "manual",
"inputSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"company\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Name of the hiring company\"\n },\n \"title\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Job title/role being advertised\"\n },\n \"location\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Work location including remote/hybrid status\"\n },\n \"type\": {\n \"type\": [\n \"string\",\n null\n ],\n \"enum\": [\n \"FULL_TIME\",\n \"PART_TIME\",\n \"CONTRACT\",\n \"INTERNSHIP\",\n \"FREELANCE\",\n null\n ],\n \"description\": \"Employment type (Full-time, Contract, etc)\"\n },\n \"work_location\": {\n \"type\": [\n \"string\",\n null\n ],\n \"enum\": [\n \"REMOTE\",\n \"HYBRID\",\n \"ON_SITE\",\n null\n ],\n \"description\": \"Work arrangement type\"\n },\n \"salary\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Compensation details if provided\"\n },\n \"description\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Main job description text including requirements and team info\"\n },\n \"apply_url\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Direct application/job posting URL\"\n },\n \"company_url\": {\n \"type\": [\n \"string\",\n null\n ],\n \"description\": \"Company website or careers page\"\n }\n }\n}\n"
},
"typeVersion": 1.2
},
{
"id": "b84ca004-6f3b-4577-8910-61b8584b161d",
"name": "Search for Who is hiring posts",
"type": "n8n-nodes-base.httpRequest",
"position": [
200,
-220
],
"parameters": {
"url": "https://uj5wyc0l7x-dsn.algolia.net/1/indexes/Item_dev_sort_date/query",
"method": "POST",
"options": {},
"jsonBody": "{\n \"query\": \"\\\"Ask HN: Who is hiring\\\"\",\n \"analyticsTags\": [\n \"web\"\n ],\n \"page\": 0,\n \"hitsPerPage\": 30,\n \"minWordSizefor1Typo\": 4,\n \"minWordSizefor2Typos\": 8,\n \"advancedSyntax\": true,\n \"ignorePlurals\": false,\n \"clickAnalytics\": true,\n \"minProximity\": 7,\n \"numericFilters\": [],\n \"tagFilters\": [\n [\n \"story\"\n ],\n []\n ],\n \"typoTolerance\": \"min\",\n \"queryType\": \"prefixNone\",\n \"restrictSearchableAttributes\": [\n \"title\",\n \"comment_text\",\n \"url\",\n \"story_text\",\n \"author\"\n ],\n \"getRankingInfo\": true\n}",
"sendBody": true,
"sendQuery": true,
"sendHeaders": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"queryParameters": {
"parameters": [
{
"name": "x-algolia-agent",
"value": "Algolia for JavaScript (4.13.1); Browser (lite)"
},
{
"name": "x-algolia-application-id",
"value": "UJ5WYC0L7X"
}
]
},
"headerParameters": {
"parameters": [
{
"name": "Accept",
"value": "*/*"
},
{
"name": "Accept-Language",
"value": "en-GB,en-US;q=0.9,en;q=0.8"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "DNT",
"value": "1"
},
{
"name": "Origin",
"value": "https://hn.algolia.com"
},
{
"name": "Referer",
"value": "https://hn.algolia.com/"
},
{
"name": "Sec-Fetch-Dest",
"value": "empty"
},
{
"name": "Sec-Fetch-Mode",
"value": "cors"
},
{
"name": "Sec-Fetch-Site",
"value": "cross-site"
},
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
},
{
"name": "sec-ch-ua",
"value": "\"Chromium\";v=\"133\", \"Not(A:Brand\";v=\"99\""
},
{
"name": "sec-ch-ua-mobile",
"value": "?0"
},
{
"name": "sec-ch-ua-platform",
"value": "\"macOS\""
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "oVEXp2ZbYCXypMVz",
"name": "Algolia Auth"
}
},
"typeVersion": 4.2
},
{
"id": "205e66f6-cd6b-4cfd-a6ec-2226c35ddaac",
"name": "Get relevant data",
"type": "n8n-nodes-base.set",
"position": [
700,
-220
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "73dd2325-faa7-4650-bd78-5fc97cc202de",
"name": "title",
"type": "string",
"value": "={{ $json.title }}"
},
{
"id": "44918eac-4510-440e-9ac0-bf14d2b2f3af",
"name": "createdAt",
"type": "string",
"value": "={{ $json.created_at }}"
},
{
"id": "00eb6f09-2c22-411c-949c-886b2d95b6eb",
"name": "updatedAt",
"type": "string",
"value": "={{ $json.updated_at }}"
},
{
"id": "2b4f9da6-f60e-46e0-ba9d-3242fa955a55",
"name": "storyId",
"type": "string",
"value": "={{ $json.story_id }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "16bc5628-8a29-4eac-8be9-b4e9da802e1e",
"name": "Get latest post",
"type": "n8n-nodes-base.filter",
"position": [
900,
-220
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "d7dd7175-2a50-45aa-bd3e-4c248c9193c4",
"operator": {
"type": "dateTime",
"operation": "after"
},
"leftValue": "={{ $json.createdAt }}",
"rightValue": "={{$now.minus({days: 30})}} "
}
]
}
},
"typeVersion": 2.2
},
{
"id": "92e1ef74-5ae1-4195-840b-115184db464f",
"name": "Split out children (jobs)",
"type": "n8n-nodes-base.splitOut",
"position": [
1460,
-220
],
"parameters": {
"options": {},
"fieldToSplitOut": "kids"
},
"typeVersion": 1
},
{
"id": "d0836aae-b98a-497f-a6f7-0ad563c262a0",
"name": "Trun into structured data",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
2600,
-220
],
"parameters": {
"text": "={{ $json.cleaned_text }}",
"messages": {
"messageValues": [
{
"message": "Extract the JSON data"
}
]
},
"promptType": "define",
"hasOutputParser": true
},
"typeVersion": 1.5
},
{
"id": "fd818a93-627c-435d-91ba-5d759d5a9004",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2600,
-620
],
"parameters": {
"width": 840,
"height": 340,
"content": "## Data Structure\n\nWe use Openai GPT-4o-mini to transform the raw data in a unified data structure. Feel free to change this.\n\n```json\n{\n \"company\": \"Name of the hiring company\",\n \"title\": \"Job title/role being advertised\",\n \"location\": \"Work location including remote/hybrid status\",\n \"type\": \"Employment type (Full-time, Contract, etc)\",\n \"salary\": \"Compensation details if provided\",\n \"description\": \"Main job description text including requirements and team info\",\n \"apply_url\": \"Direct application/job posting URL\",\n \"company_url\": \"Company website or careers page\"\n}\n```"
},
"typeVersion": 1
},
{
"id": "b70c5578-5b81-467a-8ac2-65374e4e52f3",
"name": "Extract text",
"type": "n8n-nodes-base.set",
"position": [
1860,
-220
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "6affa370-56ce-4ad8-8534-8f753fdf07fc",
"name": "text",
"type": "string",
"value": "={{ $json.text }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "acb68d88-9417-42e9-9bcc-7c2fa95c4afd",
"name": "Clean text",
"type": "n8n-nodes-base.code",
"position": [
2060,
-220
],
"parameters": {
"jsCode": "// In a Function node in n8n\nconst inputData = $input.all();\n\nfunction cleanAllPosts(data) {\n return data.map(item => {\n try {\n // Check if item exists and has the expected structure\n if (!item || typeof item !== 'object') {\n return { cleaned_text: '', error: 'Invalid item structure' };\n }\n\n // Get the text, with multiple fallbacks\n let text = '';\n if (typeof item === 'string') {\n text = item;\n } else if (item.json && item.json.text) {\n text = item.json.text;\n } else if (typeof item.json === 'string') {\n text = item.json;\n } else {\n text = JSON.stringify(item);\n }\n\n // Make sure text is a string\n text = String(text);\n \n // Perform the cleaning operations\n try {\n text = text.replace(/&#x2F;/g, '/');\n text = text.replace(/&#x27;/g, \"'\");\n text = text.replace(/&\\w+;/g, ' ');\n text = text.replace(/<[^>]*>/g, '');\n text = text.replace(/\\|\\s*/g, '| ');\n text = text.replace(/\\s+/g, ' ');\n text = text.replace(/\\s*(https?:\\/\\/[^\\s]+)\\s*/g, '\\n$1\\n');\n text = text.replace(/\\n{3,}/g, '\\n\\n');\n text = text.trim();\n } catch (cleaningError) {\n console.log('Error during text cleaning:', cleaningError);\n // Return original text if cleaning fails\n return { cleaned_text: text, warning: 'Partial cleaning applied' };\n }\n\n return { cleaned_text: text };\n \n } catch (error) {\n console.log('Error processing item:', error);\n return { \n cleaned_text: '', \n error: `Processing error: ${error.message}`,\n original: item\n };\n }\n }).filter(result => result.cleaned_text || result.error); \n}\n\ntry {\n return cleanAllPosts(inputData);\n} catch (error) {\n console.log('Fatal error:', error);\n return [{ \n cleaned_text: '', \n error: `Fatal error: ${error.message}`,\n input: inputData \n }];\n}\n"
},
"typeVersion": 2
},
{
"id": "a0727b55-565d-47c0-9ab5-0f001f4b9941",
"name": "Limit for testing (optional)",
"type": "n8n-nodes-base.limit",
"position": [
2280,
-220
],
"parameters": {
"maxItems": 5
},
"typeVersion": 1
},
{
"id": "650baf5e-c2ac-443d-8a2b-6df89717186f",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
580,
-620
],
"parameters": {
"width": 540,
"height": 340,
"content": "## Clean the result \n\n```json\n{\n\"title\": \"Ask HN: Who is hiring? (February 2025)\",\n\"createdAt\": \"2025-02-03T16:00:43Z\",\n\"updatedAt\": \"2025-02-17T08:35:44Z\",\n\"storyId\": \"42919502\"\n},\n{\n\"title\": \"Ask HN: Who is hiring? (January 2025)\",\n\"createdAt\": \"2025-01-02T16:00:09Z\",\n\"updatedAt\": \"2025-02-13T00:03:24Z\",\n\"storyId\": \"42575537\"\n},\n```"
},
"typeVersion": 1
},
{
"id": "1ca5c39f-f21d-455a-b63a-702e7e3ba02b",
"name": "Write results to airtable",
"type": "n8n-nodes-base.airtable",
"position": [
3040,
-220
],
"parameters": {
"base": {
"__rl": true,
"mode": "list",
"value": "appM2JWvA5AstsGdn",
"cachedResultUrl": "https://airtable.com/appM2JWvA5AstsGdn",
"cachedResultName": "HN Who is hiring?"
},
"table": {
"__rl": true,
"mode": "list",
"value": "tblGvcOjqbliwM7AS",
"cachedResultUrl": "https://airtable.com/appM2JWvA5AstsGdn/tblGvcOjqbliwM7AS",
"cachedResultName": "Table 1"
},
"columns": {
"value": {
"type": "={{ $json.output.type }}",
"title": "={{ $json.output.title }}",
"salary": "={{ $json.output.salary }}",
"company": "={{ $json.output.company }}",
"location": "={{ $json.output.location }}",
"apply_url": "={{ $json.output.apply_url }}",
"company_url": "={{ $json.output.company_url }}",
"description": "={{ $json.output.description }}"
},
"schema": [
{
"id": "title",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "title",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "company",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "company",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "location",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "location",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "type",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "type",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "salary",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "salary",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "description",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "description",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "apply_url",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "apply_url",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "company_url",
"type": "string",
"display": true,
"removed": false,
"readOnly": false,
"required": false,
"displayName": "company_url",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "posted_date",
"type": "string",
"display": true,
"removed": true,
"readOnly": false,
"required": false,
"displayName": "posted_date",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "create"
},
"credentials": {
"airtableTokenApi": {
"id": "IudXLNj7CDuc5M5a",
"name": "Airtable Personal Access Token account"
}
},
"typeVersion": 2.1
},
{
"id": "d71fa024-86a0-4f74-b033-1f755574080c",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-300
],
"parameters": {
"width": 380,
"height": 500,
"content": "## Hacker News - Who is Hiring Scrape\n\nIn this template we setup a scraper for the monthly HN Who is Hiring post. This way we can scrape the data and transform it to a common data strcutre.\n\nFirst we use the [Algolia Search](https://hn.algolia.com/) provided by hackernews to drill down the results.\n\nWe can use the official [Hacker News API](https://github.com/HackerNews/API\n) to get the post data and also all the replies!\n\nThis will obviously work for any kind of post on hacker news! Get creative 😃\n\nAll you need is an Openai Account to structure the text data and an Airtable Account (or similar) to write the results to a list.\n\nCopy my base https://airtable.com/appM2JWvA5AstsGdn/shrAuo78cJt5C2laR"
},
"typeVersion": 1
},
{
"id": "7466fb0c-9f0c-4adf-a6de-b2cf09032719",
"name": "HI API: Get the individual job post",
"type": "n8n-nodes-base.httpRequest",
"position": [
1660,
-220
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.kids }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "184abccf-5838-49bf-9922-e0300c6b145e",
"name": "HN API: Get Main Post",
"type": "n8n-nodes-base.httpRequest",
"position": [
1260,
-220
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.storyId }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "387f7084-58fa-4643-9351-73c870d3f028",
"connections": {
"Split Out": {
"main": [
[
{
"node": "Get relevant data",
"type": "main",
"index": 0
}
]
]
},
"Clean text": {
"main": [
[
{
"node": "Limit for testing (optional)",
"type": "main",
"index": 0
}
]
]
},
"Extract text": {
"main": [
[
{
"node": "Clean text",
"type": "main",
"index": 0
}
]
]
},
"Get latest post": {
"main": [
[
{
"node": "HN API: Get Main Post",
"type": "main",
"index": 0
}
]
]
},
"Get relevant data": {
"main": [
[
{
"node": "Get latest post",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Trun into structured data",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"HN API: Get Main Post": {
"main": [
[
{
"node": "Split out children (jobs)",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Trun into structured data",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Split out children (jobs)": {
"main": [
[
{
"node": "HI API: Get the individual job post",
"type": "main",
"index": 0
}
]
]
},
"Trun into structured data": {
"main": [
[
{
"node": "Write results to airtable",
"type": "main",
"index": 0
}
]
]
},
"Limit for testing (optional)": {
"main": [
[
{
"node": "Trun into structured data",
"type": "main",
"index": 0
}
]
]
},
"Search for Who is hiring posts": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Search for Who is hiring posts",
"type": "main",
"index": 0
}
]
]
},
"HI API: Get the individual job post": {
"main": [
[
{
"node": "Extract text",
"type": "main",
"index": 0
}
]
]
}
}
}

1498
AI_Research_RAG_and_Data_Analysis/Hacker News to Video Content.json Normal file
View File

File diff suppressed because it is too large Load Diff

1498
AI_Research_RAG_and_Data_Analysis/Hacker News to Video Content.txt
View File

File diff suppressed because it is too large Load Diff

2942
AI_Research_RAG_and_Data_Analysis/Host Your Own AI Deep Research Agent with n8n, Apify and OpenAI o3.json Normal file
View File

File diff suppressed because it is too large Load Diff

5883
AI_Research_RAG_and_Data_Analysis/Host Your Own AI Deep Research Agent with n8n, Apify and OpenAI o3.txt
View File

File diff suppressed because it is too large Load Diff

576
AI_Research_RAG_and_Data_Analysis/Intelligent Web Query and Semantic Re-Ranking Flow using Brave and Google Gemini.json Normal file
View File

File diff suppressed because one or more lines are too long

576
AI_Research_RAG_and_Data_Analysis/Intelligent Web Query and Semantic Re-Ranking Flow using Brave and Google Gemini.txt
View File

File diff suppressed because one or more lines are too long

295
AI_Research_RAG_and_Data_Analysis/Learn Anything from HN - Get Top Resource Recommendations from Hacker News.json Normal file
View File

@@ -0,0 +1,295 @@
{
"nodes": [
{
"id": "41183066-0045-4a75-ba23-42f4efcfeccc",
"name": "Google Gemini Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatGoogleGemini",
"position": [
720,
720
],
"parameters": {
"options": {},
"modelName": "models/gemini-1.5-flash"
},
"credentials": {
"googlePalmApi": {
"id": "Hx1fn2jrUvojSKye",
"name": "Google Gemini(PaLM) Api account"
}
},
"typeVersion": 1
},
{
"id": "eb061c39-7a4d-42e7-bb42-806504731b11",
"name": "Basic LLM Chain",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
700,
560
],
"parameters": {
"text": "=Your Task is to find the best resources to learn {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}. \n\nI have scraped the HackerNews and The following is the list of comments from HackerNews on topic about Learning {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}\n\n\nFocus only on comments that provide any resouces or advice or insight about learning {{ $('GetTopicFromToLearn').item.json.Learn }}. Ignore all other comments that are off topic discussions.\n\nNow based on these comments, you need to find the top resources and list them. \n\nCategorize them based on resource type (course, book, article, youtube videos, lectures, etc) and also figure out the difficultiy level (beginner, intermediate, advanced, expert).\n\nYou don't always to have fill in these categories exactly, these are given here for reference. Use your intution to find the best categorization.\n\nNow based on these metrics and running a basic sentiment analysis on comments you need to figure out what the top resources are. \n\nRespond back in Markdown formatted text. In the following format\n\n**OUTPUT FORMAT**\n\n```\n\n## Top HN Recomended Resources To Learn <topic Name> \n\n### Category 1\n\n- **Resource 1** - One line description\n- **Resource 2** - One line description\n- ... \n\n<add hyperlinks if any exists>\n\n### Category 2\n\n- **Resource 1** - One line description\n- **Resource 2** - One line description\n- ... \n\n<add hyperlinks in markdown format to the resource name itself if any exists. Example [resource name](https://example.com)>\n\n...\n```\n\nHere is the list of HackerNews Comments.\n\n{{ $json.text }}",
"promptType": "define"
},
"typeVersion": 1.5
},
{
"id": "94073fe0-d25c-421e-9c99-67b6c4f0afad",
"name": "SearchAskHN",
"type": "n8n-nodes-base.hackerNews",
"position": [
-160,
560
],
"parameters": {
"limit": 150,
"resource": "all",
"additionalFields": {
"tags": [
"ask_hn"
],
"keyword": "={{ $json[\"I want to learn\"] }}"
}
},
"typeVersion": 1
},
{
"id": "eee4dfdf-53ab-42be-91ae-7b6c405df7c2",
"name": "FindHNComments",
"type": "n8n-nodes-base.httpRequest",
"position": [
260,
560
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.children }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "e57d86ae-d7c1-4354-9e3c-528c76160cd9",
"name": "CombineIntoSingleText",
"type": "n8n-nodes-base.aggregate",
"position": [
480,
560
],
"parameters": {
"options": {},
"fieldsToAggregate": {
"fieldToAggregate": [
{
"fieldToAggregate": "text"
}
]
}
},
"typeVersion": 1
},
{
"id": "b2086d29-1de5-48f4-8c1e-affd509fb5f7",
"name": "SplitOutChildrenIDs",
"type": "n8n-nodes-base.splitOut",
"position": [
40,
560
],
"parameters": {
"options": {},
"fieldToSplitOut": "children"
},
"typeVersion": 1
},
{
"id": "6fe68a4b-744b-48c8-9320-d2b19e3eb92b",
"name": "GetTopicFromToLearn",
"type": "n8n-nodes-base.formTrigger",
"position": [
-340,
560
],
"webhookId": "4524d82f-86a6-4fab-ba09-1d24001e15f3",
"parameters": {
"options": {
"path": "learn",
"buttonLabel": "Submit",
"respondWithOptions": {
"values": {
"formSubmittedText": "We'll shortly send you an email with top recommendations."
}
}
},
"formTitle": "What do You want to learn ?",
"formFields": {
"values": [
{
"fieldLabel": "I want to learn",
"placeholder": "Python, DevOps, Ai, or just about anything"
},
{
"fieldType": "email",
"fieldLabel": "What's your email ?",
"placeholder": "john.doe@example.com",
"requiredField": true
}
]
},
"formDescription": "We'll find the best resources from HackerNews and send you an email"
},
"typeVersion": 2.2
},
{
"id": "72fcb7f3-6706-47cc-8a79-364b325aa8ae",
"name": "SendEmailWithTopResources",
"type": "n8n-nodes-base.emailSend",
"position": [
1320,
560
],
"parameters": {
"html": "=FYI, We read through {{ $('SplitOutChildrenIDs').all().length }} comments in search for the best.\n\n{{ $json.data }}",
"options": {},
"subject": "=Here are Top HN Recommendations for Learning {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}",
"toEmail": "={{ $('GetTopicFromToLearn').item.json[\"What's your email ?\"] }}",
"fromEmail": "allsmallnocaps@gmail.com"
},
"credentials": {
"smtp": {
"id": "knhWxmnfY16ZQwBm",
"name": "allsamll Gmail SMTP account"
}
},
"typeVersion": 2.1
},
{
"id": "b4d50b42-9e40-46b0-a411-90210b422de3",
"name": "Convert2HTML",
"type": "n8n-nodes-base.markdown",
"position": [
1100,
560
],
"parameters": {
"mode": "markdownToHtml",
"options": {},
"markdown": "={{ $json.text }}"
},
"typeVersion": 1
},
{
"id": "b79e867a-ea3b-4a94-9809-b5a01ee2820f",
"name": "Finished",
"type": "n8n-nodes-base.noOp",
"position": [
1540,
560
],
"parameters": {},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"SearchAskHN": {
"main": [
[
{
"node": "SplitOutChildrenIDs",
"type": "main",
"index": 0
}
]
]
},
"Convert2HTML": {
"main": [
[
{
"node": "SendEmailWithTopResources",
"type": "main",
"index": 0
}
]
]
},
"FindHNComments": {
"main": [
[
{
"node": "CombineIntoSingleText",
"type": "main",
"index": 0
}
]
]
},
"Basic LLM Chain": {
"main": [
[
{
"node": "Convert2HTML",
"type": "main",
"index": 0
}
]
]
},
"GetTopicFromToLearn": {
"main": [
[
{
"node": "SearchAskHN",
"type": "main",
"index": 0
}
]
]
},
"SplitOutChildrenIDs": {
"main": [
[
{
"node": "FindHNComments",
"type": "main",
"index": 0
}
]
]
},
"CombineIntoSingleText": {
"main": [
[
{
"node": "Basic LLM Chain",
"type": "main",
"index": 0
}
]
]
},
"Google Gemini Chat Model": {
"ai_languageModel": [
[
{
"node": "Basic LLM Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"SendEmailWithTopResources": {
"main": [
[
{
"node": "Finished",
"type": "main",
"index": 0
}
]
]
}
}
}

295
AI_Research_RAG_and_Data_Analysis/Learn Anything from HN - Get Top Resource Recommendations from Hacker News.txt
View File

@@ -1,295 +0,0 @@
{
"nodes": [
{
"id": "41183066-0045-4a75-ba23-42f4efcfeccc",
"name": "Google Gemini Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatGoogleGemini",
"position": [
720,
720
],
"parameters": {
"options": {},
"modelName": "models/gemini-1.5-flash"
},
"credentials": {
"googlePalmApi": {
"id": "Hx1fn2jrUvojSKye",
"name": "Google Gemini(PaLM) Api account"
}
},
"typeVersion": 1
},
{
"id": "eb061c39-7a4d-42e7-bb42-806504731b11",
"name": "Basic LLM Chain",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
700,
560
],
"parameters": {
"text": "=Your Task is to find the best resources to learn {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}. \n\nI have scraped the HackerNews and The following is the list of comments from HackerNews on topic about Learning {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}\n\n\nFocus only on comments that provide any resouces or advice or insight about learning {{ $('GetTopicFromToLearn').item.json.Learn }}. Ignore all other comments that are off topic discussions.\n\nNow based on these comments, you need to find the top resources and list them. \n\nCategorize them based on resource type (course, book, article, youtube videos, lectures, etc) and also figure out the difficultiy level (beginner, intermediate, advanced, expert).\n\nYou don't always to have fill in these categories exactly, these are given here for reference. Use your intution to find the best categorization.\n\nNow based on these metrics and running a basic sentiment analysis on comments you need to figure out what the top resources are. \n\nRespond back in Markdown formatted text. In the following format\n\n**OUTPUT FORMAT**\n\n```\n\n## Top HN Recomended Resources To Learn <topic Name> \n\n### Category 1\n\n- **Resource 1** - One line description\n- **Resource 2** - One line description\n- ... \n\n<add hyperlinks if any exists>\n\n### Category 2\n\n- **Resource 1** - One line description\n- **Resource 2** - One line description\n- ... \n\n<add hyperlinks in markdown format to the resource name itself if any exists. Example [resource name](https://example.com)>\n\n...\n```\n\nHere is the list of HackerNews Comments.\n\n{{ $json.text }}",
"promptType": "define"
},
"typeVersion": 1.5
},
{
"id": "94073fe0-d25c-421e-9c99-67b6c4f0afad",
"name": "SearchAskHN",
"type": "n8n-nodes-base.hackerNews",
"position": [
-160,
560
],
"parameters": {
"limit": 150,
"resource": "all",
"additionalFields": {
"tags": [
"ask_hn"
],
"keyword": "={{ $json[\"I want to learn\"] }}"
}
},
"typeVersion": 1
},
{
"id": "eee4dfdf-53ab-42be-91ae-7b6c405df7c2",
"name": "FindHNComments",
"type": "n8n-nodes-base.httpRequest",
"position": [
260,
560
],
"parameters": {
"url": "=https://hacker-news.firebaseio.com/v0/item/{{ $json.children }}.json?print=pretty",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "e57d86ae-d7c1-4354-9e3c-528c76160cd9",
"name": "CombineIntoSingleText",
"type": "n8n-nodes-base.aggregate",
"position": [
480,
560
],
"parameters": {
"options": {},
"fieldsToAggregate": {
"fieldToAggregate": [
{
"fieldToAggregate": "text"
}
]
}
},
"typeVersion": 1
},
{
"id": "b2086d29-1de5-48f4-8c1e-affd509fb5f7",
"name": "SplitOutChildrenIDs",
"type": "n8n-nodes-base.splitOut",
"position": [
40,
560
],
"parameters": {
"options": {},
"fieldToSplitOut": "children"
},
"typeVersion": 1
},
{
"id": "6fe68a4b-744b-48c8-9320-d2b19e3eb92b",
"name": "GetTopicFromToLearn",
"type": "n8n-nodes-base.formTrigger",
"position": [
-340,
560
],
"webhookId": "4524d82f-86a6-4fab-ba09-1d24001e15f3",
"parameters": {
"options": {
"path": "learn",
"buttonLabel": "Submit",
"respondWithOptions": {
"values": {
"formSubmittedText": "We'll shortly send you an email with top recommendations."
}
}
},
"formTitle": "What do You want to learn ?",
"formFields": {
"values": [
{
"fieldLabel": "I want to learn",
"placeholder": "Python, DevOps, Ai, or just about anything"
},
{
"fieldType": "email",
"fieldLabel": "What's your email ?",
"placeholder": "john.doe@example.com",
"requiredField": true
}
]
},
"formDescription": "We'll find the best resources from HackerNews and send you an email"
},
"typeVersion": 2.2
},
{
"id": "72fcb7f3-6706-47cc-8a79-364b325aa8ae",
"name": "SendEmailWithTopResources",
"type": "n8n-nodes-base.emailSend",
"position": [
1320,
560
],
"parameters": {
"html": "=FYI, We read through {{ $('SplitOutChildrenIDs').all().length }} comments in search for the best.\n\n{{ $json.data }}",
"options": {},
"subject": "=Here are Top HN Recommendations for Learning {{ $('GetTopicFromToLearn').item.json[\"I want to learn\"] }}",
"toEmail": "={{ $('GetTopicFromToLearn').item.json[\"What's your email ?\"] }}",
"fromEmail": "allsmallnocaps@gmail.com"
},
"credentials": {
"smtp": {
"id": "knhWxmnfY16ZQwBm",
"name": "allsamll Gmail SMTP account"
}
},
"typeVersion": 2.1
},
{
"id": "b4d50b42-9e40-46b0-a411-90210b422de3",
"name": "Convert2HTML",
"type": "n8n-nodes-base.markdown",
"position": [
1100,
560
],
"parameters": {
"mode": "markdownToHtml",
"options": {},
"markdown": "={{ $json.text }}"
},
"typeVersion": 1
},
{
"id": "b79e867a-ea3b-4a94-9809-b5a01ee2820f",
"name": "Finished",
"type": "n8n-nodes-base.noOp",
"position": [
1540,
560
],
"parameters": {},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"SearchAskHN": {
"main": [
[
{
"node": "SplitOutChildrenIDs",
"type": "main",
"index": 0
}
]
]
},
"Convert2HTML": {
"main": [
[
{
"node": "SendEmailWithTopResources",
"type": "main",
"index": 0
}
]
]
},
"FindHNComments": {
"main": [
[
{
"node": "CombineIntoSingleText",
"type": "main",
"index": 0
}
]
]
},
"Basic LLM Chain": {
"main": [
[
{
"node": "Convert2HTML",
"type": "main",
"index": 0
}
]
]
},
"GetTopicFromToLearn": {
"main": [
[
{
"node": "SearchAskHN",
"type": "main",
"index": 0
}
]
]
},
"SplitOutChildrenIDs": {
"main": [
[
{
"node": "FindHNComments",
"type": "main",
"index": 0
}
]
]
},
"CombineIntoSingleText": {
"main": [
[
{
"node": "Basic LLM Chain",
"type": "main",
"index": 0
}
]
]
},
"Google Gemini Chat Model": {
"ai_languageModel": [
[
{
"node": "Basic LLM Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"SendEmailWithTopResources": {
"main": [
[
{
"node": "Finished",
"type": "main",
"index": 0
}
]
]
}
}
}

502
AI_Research_RAG_and_Data_Analysis/Make OpenAI Citation for File Retrieval RAG.json Normal file
View File

@@ -0,0 +1,502 @@
{
"id": "5NAbfX550LJsfz6f",
"meta": {
"instanceId": "00493e38fecfc163cb182114bc2fab90114038eb9aad665a7a752d076920d3d5",
"templateCredsSetupCompleted": true
},
"name": "Make OpenAI Citation for File Retrieval RAG",
"tags": [
{
"id": "urxRtGxxLObZWPvX",
"name": "sample",
"createdAt": "2024-09-13T02:43:13.014Z",
"updatedAt": "2024-09-13T02:43:13.014Z"
},
{
"id": "nMXS3c9l1WqDwWF5",
"name": "assist",
"createdAt": "2024-12-23T16:09:38.737Z",
"updatedAt": "2024-12-23T16:09:38.737Z"
}
],
"nodes": [
{
"id": "b9033511-3421-467a-9bfa-73af01b99c4f",
"name": "Aggregate",
"type": "n8n-nodes-base.aggregate",
"position": [
740,
120
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "a61dd9d3-4faa-4878-a6f3-ba8277279002",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
980,
-320
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "2daabca5-37ec-4cad-9157-29926367e1a7",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
320
],
"parameters": {
"color": 3,
"width": 840,
"height": 80,
"content": "## Within N8N, there will be a chat button to test"
},
"typeVersion": 1
},
{
"id": "bf4485b1-cd94-41c8-a183-bf1b785f2761",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-440,
-520
],
"parameters": {
"color": 5,
"width": 500,
"height": 720,
"content": "## Make OpenAI Citation for File Retrieval RAG\n\n## Use case\n\nIn this example, we will ensure that all texts from the OpenAI assistant search for citations and sources in the vector store files. We can also format the output for Markdown or HTML tags.\n\nThis is necessary because the assistant sometimes generates strange characters, and we can also use dynamic references such as citations 1, 2, 3, for example.\n\n## What this workflow does\n\nIn this workflow, we will use an OpenAI assistant created within their interface, equipped with a vector store containing some files for file retrieval.\n\nThe assistant will perform the file search within the OpenAI infrastructure and will return the content with citations.\n\n- We will make an HTTP request to retrieve all the details we need to format the text output.\n\n## Setup\n\nInsert an OpenAI Key\n\n## How to adjust it to your needs\n\nAt the end of the workflow, we have a block of code that will format the output, and there we can add Markdown tags to create links. Optionally, we can transform the Markdown formatting into HTML.\n\n\nby Davi Saranszky Mesquita\nhttps://www.linkedin.com/in/mesquitadavi/"
},
"typeVersion": 1
},
{
"id": "539a4e40-9745-4a26-aba8-2cc2b0dd6364",
"name": "Create a simple Trigger to have the Chat button within N8N",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"notes": "https://www.npmjs.com/package/@n8n/chat",
"position": [
260,
-520
],
"webhookId": "8ccaa299-6f99-427b-9356-e783893a3d0c",
"parameters": {
"options": {}
},
"notesInFlow": true,
"typeVersion": 1.1
},
{
"id": "aa5b2951-df32-43ac-9939-83b02d818e73",
"name": "OpenAI Assistant with Vector Store",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
580,
-520
],
"parameters": {
"options": {
"preserveOriginalTools": false
},
"resource": "assistant",
"assistantId": {
"__rl": true,
"mode": "list",
"value": "asst_QAfdobVCVCMJz8LmaEC7nlId",
"cachedResultName": "Teste"
}
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 1.7
},
{
"id": "1817b673-6cb3-49aa-9f38-a5876eb0e6fa",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
560,
-680
],
"parameters": {
"width": 300,
"content": "## Setup\n\n- Configure OpenAI Key\n\n### In this step, we will use an assistant created within the OpenAI platform that contains a vector store a.k.a file retrieval"
},
"typeVersion": 1
},
{
"id": "16429226-e850-4698-b419-fd9805a03fb7",
"name": "Get ALL Thread Content",
"type": "n8n-nodes-base.httpRequest",
"position": [
1260,
-520
],
"parameters": {
"url": "=https://api.openai.com/v1/threads/{{ $json.threadId }}/messages",
"options": {},
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"headerParameters": {
"parameters": [
{
"name": "OpenAI-Beta",
"value": "assistants=v2"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 4.2,
"alwaysOutputData": true
},
{
"id": "e8c88b08-5be2-4f7e-8b17-8cf804b3fe9f",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1160,
-620
],
"parameters": {
"content": "### Retrieving all thread content is necessary because the OpenAI tool does not retrieve all citations upon request."
},
"typeVersion": 1
},
{
"id": "0f51e09f-2782-4e2d-b797-d4d58fcabdaf",
"name": "Split all message iterations from a thread",
"type": "n8n-nodes-base.splitOut",
"position": [
220,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "data"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "4d569993-1ce3-4b32-beaf-382feac25da9",
"name": "Split all content from a single message",
"type": "n8n-nodes-base.splitOut",
"position": [
460,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "content"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "999e1c2b-1927-4483-aac1-6e8903f7ed25",
"name": "Split all citations from a single message",
"type": "n8n-nodes-base.splitOut",
"position": [
700,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "text.annotations"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "98af62f5-adb0-4e07-a146-fc2f13b851ce",
"name": "Retrieve file name from a file ID",
"type": "n8n-nodes-base.httpRequest",
"onError": "continueRegularOutput",
"position": [
220,
120
],
"parameters": {
"url": "=https://api.openai.com/v1/files/{{ $json.file_citation.file_id }}",
"options": {},
"sendQuery": true,
"authentication": "predefinedCredentialType",
"queryParameters": {
"parameters": [
{
"name": "limit",
"value": "1"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 4.2,
"alwaysOutputData": true
},
{
"id": "b11f0d3d-bdc4-4845-b14b-d0b0de214f01",
"name": "Regularize output",
"type": "n8n-nodes-base.set",
"position": [
480,
120
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "2dcaafee-5037-4a97-942a-bcdd02bc2ad9",
"name": "id",
"type": "string",
"value": "={{ $json.id }}"
},
{
"id": "b63f967d-ceea-4aa8-98b9-91f5ab21bfe8",
"name": "filename",
"type": "string",
"value": "={{ $json.filename }}"
},
{
"id": "f611e749-054a-441d-8610-df8ba42de2e1",
"name": "text",
"type": "string",
"value": "={{ $('Split all citations from a single message').item.json.text }}"
}
]
}
},
"typeVersion": 3.4,
"alwaysOutputData": true
},
{
"id": "0e999a0e-76ed-4897-989b-228f075e9bfb",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-60
],
"parameters": {
"width": 200,
"height": 220,
"content": "### A file retrieval request contains a lot of information, and we want only the text that will be substituted and the file name.\n\n- id\n- filename\n- text\n"
},
"typeVersion": 1
},
{
"id": "53c79a6c-7543-435f-b40e-966dff0904d4",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
700,
-60
],
"parameters": {
"width": 200,
"height": 220,
"content": "### With the last three splits, we may have many citations and texts to substitute. By doing an aggregation, it will be possible to handle everything as a single request."
},
"typeVersion": 1
},
{
"id": "381fb6d6-64fc-4668-9d3c-98aaa43a45ca",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
960,
-60
],
"parameters": {
"height": 220,
"content": "### This simple code will take all the previous files and citations and alter the original text, formatting the output. In this way, we can use Markdown tags to create links, or if you prefer, we can add an HTML transformation node."
},
"typeVersion": 1
},
{
"id": "d0cbb943-57ab-4850-8370-1625610a852a",
"name": "Optional Markdown to HTML",
"type": "n8n-nodes-base.markdown",
"disabled": true,
"position": [
1220,
120
],
"parameters": {
"html": "={{ $json.output }}",
"options": {},
"destinationKey": "output"
},
"typeVersion": 1
},
{
"id": "589e2418-5dec-47d0-ba08-420d84f09da7",
"name": "Finnaly format the output",
"type": "n8n-nodes-base.code",
"position": [
980,
120
],
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "let saida = $('OpenAI Assistant with Vector Store').item.json.output;\n\nfor (let i of $input.item.json.data) {\n saida = saida.replaceAll(i.text, \" _(\"+ i.filename+\")_ \");\n}\n\n$input.item.json.output = saida;\nreturn $input.item;"
},
"typeVersion": 2
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "0e621a5a-d99d-4db3-9ae4-ea98c31467e9",
"connections": {
"Aggregate": {
"main": [
[
{
"node": "Finnaly format the output",
"type": "main",
"index": 0
}
]
]
},
"Regularize output": {
"main": [
[
{
"node": "Aggregate",
"type": "main",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "OpenAI Assistant with Vector Store",
"type": "ai_memory",
"index": 0
}
]
]
},
"Get ALL Thread Content": {
"main": [
[
{
"node": "Split all message iterations from a thread",
"type": "main",
"index": 0
}
]
]
},
"Finnaly format the output": {
"main": [
[
{
"node": "Optional Markdown to HTML",
"type": "main",
"index": 0
}
]
]
},
"Retrieve file name from a file ID": {
"main": [
[
{
"node": "Regularize output",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Assistant with Vector Store": {
"main": [
[
{
"node": "Get ALL Thread Content",
"type": "main",
"index": 0
}
]
]
},
"Split all content from a single message": {
"main": [
[
{
"node": "Split all citations from a single message",
"type": "main",
"index": 0
}
]
]
},
"Split all citations from a single message": {
"main": [
[
{
"node": "Retrieve file name from a file ID",
"type": "main",
"index": 0
}
]
]
},
"Split all message iterations from a thread": {
"main": [
[
{
"node": "Split all content from a single message",
"type": "main",
"index": 0
}
]
]
},
"Create a simple Trigger to have the Chat button within N8N": {
"main": [
[
{
"node": "OpenAI Assistant with Vector Store",
"type": "main",
"index": 0
}
]
]
}
}
}

502
AI_Research_RAG_and_Data_Analysis/Make OpenAI Citation for File Retrieval RAG.txt
View File

@@ -1,502 +0,0 @@
{
"id": "5NAbfX550LJsfz6f",
"meta": {
"instanceId": "00493e38fecfc163cb182114bc2fab90114038eb9aad665a7a752d076920d3d5",
"templateCredsSetupCompleted": true
},
"name": "Make OpenAI Citation for File Retrieval RAG",
"tags": [
{
"id": "urxRtGxxLObZWPvX",
"name": "sample",
"createdAt": "2024-09-13T02:43:13.014Z",
"updatedAt": "2024-09-13T02:43:13.014Z"
},
{
"id": "nMXS3c9l1WqDwWF5",
"name": "assist",
"createdAt": "2024-12-23T16:09:38.737Z",
"updatedAt": "2024-12-23T16:09:38.737Z"
}
],
"nodes": [
{
"id": "b9033511-3421-467a-9bfa-73af01b99c4f",
"name": "Aggregate",
"type": "n8n-nodes-base.aggregate",
"position": [
740,
120
],
"parameters": {
"options": {},
"aggregate": "aggregateAllItemData"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "a61dd9d3-4faa-4878-a6f3-ba8277279002",
"name": "Window Buffer Memory",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
980,
-320
],
"parameters": {},
"typeVersion": 1.3
},
{
"id": "2daabca5-37ec-4cad-9157-29926367e1a7",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
320
],
"parameters": {
"color": 3,
"width": 840,
"height": 80,
"content": "## Within N8N, there will be a chat button to test"
},
"typeVersion": 1
},
{
"id": "bf4485b1-cd94-41c8-a183-bf1b785f2761",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-440,
-520
],
"parameters": {
"color": 5,
"width": 500,
"height": 720,
"content": "## Make OpenAI Citation for File Retrieval RAG\n\n## Use case\n\nIn this example, we will ensure that all texts from the OpenAI assistant search for citations and sources in the vector store files. We can also format the output for Markdown or HTML tags.\n\nThis is necessary because the assistant sometimes generates strange characters, and we can also use dynamic references such as citations 1, 2, 3, for example.\n\n## What this workflow does\n\nIn this workflow, we will use an OpenAI assistant created within their interface, equipped with a vector store containing some files for file retrieval.\n\nThe assistant will perform the file search within the OpenAI infrastructure and will return the content with citations.\n\n- We will make an HTTP request to retrieve all the details we need to format the text output.\n\n## Setup\n\nInsert an OpenAI Key\n\n## How to adjust it to your needs\n\nAt the end of the workflow, we have a block of code that will format the output, and there we can add Markdown tags to create links. Optionally, we can transform the Markdown formatting into HTML.\n\n\nby Davi Saranszky Mesquita\nhttps://www.linkedin.com/in/mesquitadavi/"
},
"typeVersion": 1
},
{
"id": "539a4e40-9745-4a26-aba8-2cc2b0dd6364",
"name": "Create a simple Trigger to have the Chat button within N8N",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"notes": "https://www.npmjs.com/package/@n8n/chat",
"position": [
260,
-520
],
"webhookId": "8ccaa299-6f99-427b-9356-e783893a3d0c",
"parameters": {
"options": {}
},
"notesInFlow": true,
"typeVersion": 1.1
},
{
"id": "aa5b2951-df32-43ac-9939-83b02d818e73",
"name": "OpenAI Assistant with Vector Store",
"type": "@n8n/n8n-nodes-langchain.openAi",
"position": [
580,
-520
],
"parameters": {
"options": {
"preserveOriginalTools": false
},
"resource": "assistant",
"assistantId": {
"__rl": true,
"mode": "list",
"value": "asst_QAfdobVCVCMJz8LmaEC7nlId",
"cachedResultName": "Teste"
}
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 1.7
},
{
"id": "1817b673-6cb3-49aa-9f38-a5876eb0e6fa",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
560,
-680
],
"parameters": {
"width": 300,
"content": "## Setup\n\n- Configure OpenAI Key\n\n### In this step, we will use an assistant created within the OpenAI platform that contains a vector store a.k.a file retrieval"
},
"typeVersion": 1
},
{
"id": "16429226-e850-4698-b419-fd9805a03fb7",
"name": "Get ALL Thread Content",
"type": "n8n-nodes-base.httpRequest",
"position": [
1260,
-520
],
"parameters": {
"url": "=https://api.openai.com/v1/threads/{{ $json.threadId }}/messages",
"options": {},
"sendHeaders": true,
"authentication": "predefinedCredentialType",
"headerParameters": {
"parameters": [
{
"name": "OpenAI-Beta",
"value": "assistants=v2"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 4.2,
"alwaysOutputData": true
},
{
"id": "e8c88b08-5be2-4f7e-8b17-8cf804b3fe9f",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1160,
-620
],
"parameters": {
"content": "### Retrieving all thread content is necessary because the OpenAI tool does not retrieve all citations upon request."
},
"typeVersion": 1
},
{
"id": "0f51e09f-2782-4e2d-b797-d4d58fcabdaf",
"name": "Split all message iterations from a thread",
"type": "n8n-nodes-base.splitOut",
"position": [
220,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "data"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "4d569993-1ce3-4b32-beaf-382feac25da9",
"name": "Split all content from a single message",
"type": "n8n-nodes-base.splitOut",
"position": [
460,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "content"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "999e1c2b-1927-4483-aac1-6e8903f7ed25",
"name": "Split all citations from a single message",
"type": "n8n-nodes-base.splitOut",
"position": [
700,
-300
],
"parameters": {
"options": {},
"fieldToSplitOut": "text.annotations"
},
"typeVersion": 1,
"alwaysOutputData": true
},
{
"id": "98af62f5-adb0-4e07-a146-fc2f13b851ce",
"name": "Retrieve file name from a file ID",
"type": "n8n-nodes-base.httpRequest",
"onError": "continueRegularOutput",
"position": [
220,
120
],
"parameters": {
"url": "=https://api.openai.com/v1/files/{{ $json.file_citation.file_id }}",
"options": {},
"sendQuery": true,
"authentication": "predefinedCredentialType",
"queryParameters": {
"parameters": [
{
"name": "limit",
"value": "1"
}
]
},
"nodeCredentialType": "openAiApi"
},
"credentials": {
"openAiApi": {
"id": "UfNrqPCRlD8FD9mk",
"name": "OpenAi Lourival"
}
},
"typeVersion": 4.2,
"alwaysOutputData": true
},
{
"id": "b11f0d3d-bdc4-4845-b14b-d0b0de214f01",
"name": "Regularize output",
"type": "n8n-nodes-base.set",
"position": [
480,
120
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "2dcaafee-5037-4a97-942a-bcdd02bc2ad9",
"name": "id",
"type": "string",
"value": "={{ $json.id }}"
},
{
"id": "b63f967d-ceea-4aa8-98b9-91f5ab21bfe8",
"name": "filename",
"type": "string",
"value": "={{ $json.filename }}"
},
{
"id": "f611e749-054a-441d-8610-df8ba42de2e1",
"name": "text",
"type": "string",
"value": "={{ $('Split all citations from a single message').item.json.text }}"
}
]
}
},
"typeVersion": 3.4,
"alwaysOutputData": true
},
{
"id": "0e999a0e-76ed-4897-989b-228f075e9bfb",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-60
],
"parameters": {
"width": 200,
"height": 220,
"content": "### A file retrieval request contains a lot of information, and we want only the text that will be substituted and the file name.\n\n- id\n- filename\n- text\n"
},
"typeVersion": 1
},
{
"id": "53c79a6c-7543-435f-b40e-966dff0904d4",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
700,
-60
],
"parameters": {
"width": 200,
"height": 220,
"content": "### With the last three splits, we may have many citations and texts to substitute. By doing an aggregation, it will be possible to handle everything as a single request."
},
"typeVersion": 1
},
{
"id": "381fb6d6-64fc-4668-9d3c-98aaa43a45ca",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
960,
-60
],
"parameters": {
"height": 220,
"content": "### This simple code will take all the previous files and citations and alter the original text, formatting the output. In this way, we can use Markdown tags to create links, or if you prefer, we can add an HTML transformation node."
},
"typeVersion": 1
},
{
"id": "d0cbb943-57ab-4850-8370-1625610a852a",
"name": "Optional Markdown to HTML",
"type": "n8n-nodes-base.markdown",
"disabled": true,
"position": [
1220,
120
],
"parameters": {
"html": "={{ $json.output }}",
"options": {},
"destinationKey": "output"
},
"typeVersion": 1
},
{
"id": "589e2418-5dec-47d0-ba08-420d84f09da7",
"name": "Finnaly format the output",
"type": "n8n-nodes-base.code",
"position": [
980,
120
],
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "let saida = $('OpenAI Assistant with Vector Store').item.json.output;\n\nfor (let i of $input.item.json.data) {\n saida = saida.replaceAll(i.text, \" _(\"+ i.filename+\")_ \");\n}\n\n$input.item.json.output = saida;\nreturn $input.item;"
},
"typeVersion": 2
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "0e621a5a-d99d-4db3-9ae4-ea98c31467e9",
"connections": {
"Aggregate": {
"main": [
[
{
"node": "Finnaly format the output",
"type": "main",
"index": 0
}
]
]
},
"Regularize output": {
"main": [
[
{
"node": "Aggregate",
"type": "main",
"index": 0
}
]
]
},
"Window Buffer Memory": {
"ai_memory": [
[
{
"node": "OpenAI Assistant with Vector Store",
"type": "ai_memory",
"index": 0
}
]
]
},
"Get ALL Thread Content": {
"main": [
[
{
"node": "Split all message iterations from a thread",
"type": "main",
"index": 0
}
]
]
},
"Finnaly format the output": {
"main": [
[
{
"node": "Optional Markdown to HTML",
"type": "main",
"index": 0
}
]
]
},
"Retrieve file name from a file ID": {
"main": [
[
{
"node": "Regularize output",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Assistant with Vector Store": {
"main": [
[
{
"node": "Get ALL Thread Content",
"type": "main",
"index": 0
}
]
]
},
"Split all content from a single message": {
"main": [
[
{
"node": "Split all citations from a single message",
"type": "main",
"index": 0
}
]
]
},
"Split all citations from a single message": {
"main": [
[
{
"node": "Retrieve file name from a file ID",
"type": "main",
"index": 0
}
]
]
},
"Split all message iterations from a thread": {
"main": [
[
{
"node": "Split all content from a single message",
"type": "main",
"index": 0
}
]
]
},
"Create a simple Trigger to have the Chat button within N8N": {
"main": [
[
{
"node": "OpenAI Assistant with Vector Store",
"type": "main",
"index": 0
}
]
]
}
}
}

468
AI_Research_RAG_and_Data_Analysis/Open Deep Research - AI-Powered Autonomous Research Workflow.json Normal file
View File

@@ -0,0 +1,468 @@
{
"id": "WLSqXECfQF7rOj2A",
"meta": {
"instanceId": "cba4a4a2eb5d7683330e2944837278938831ed3c042e20da6f5049c07ad14798"
},
"name": "Open Deep Research - AI-Powered Autonomous Research Workflow",
"tags": [],
"nodes": [
{
"id": "b7b70ba1-0267-4d2b-91f4-5cc4fd22fd03",
"name": "Chat Message Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
-1940,
160
],
"webhookId": "cb0b9dbe-1f35-441a-b062-29624b0ebc6a",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "55a8a512-f2d4-4aed-93e5-dd9bfa2dcaad",
"name": "Generate Search Queries using LLM",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
-1760,
160
],
"parameters": {
"text": "=User Query: {{ $('Chat Message Trigger').item.json.chatInput }}",
"messages": {
"messageValues": [
{
"message": "=You are an expert research assistant. Given a user's query, generate up to four distinct, precise search queries that would help gather comprehensive information on the topic. Return only a JSON list of strings, for example: ['query1', 'query2', 'query3']."
}
]
},
"promptType": "define"
},
"typeVersion": 1.5
},
{
"id": "5f92361a-b490-479d-8360-c87a100b470e",
"name": "LLM Response Provider (OpenRouter)",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter",
"position": [
-1760,
700
],
"parameters": {
"model": "google/gemini-2.0-flash-001",
"options": {}
},
"credentials": {
"openRouterApi": {
"id": "WZWYWCfluxuKxZzV",
"name": "OpenRouter account"
}
},
"typeVersion": 1
},
{
"id": "4ab360eb-858f-48b8-a00d-71867d4f0c93",
"name": "Parse and Chunk JSON Data",
"type": "n8n-nodes-base.code",
"position": [
-1420,
160
],
"parameters": {
"jsCode": "// Parse the input JSON string and split it into four chunks\nconst rawText = $json.text;\n\n// Remove Markdown JSON code blocks if present\nconst cleanedText = rawText.replace(/```json|```/g, '').trim();\n\ntry {\n const jsonArray = JSON.parse(cleanedText);\n if (!Array.isArray(jsonArray)) {\n throw new Error('The JSON is not an array.');\n }\n const chunkSize = Math.ceil(jsonArray.length / 4);\n const chunks = [];\n for (let i = 0; i < jsonArray.length; i += chunkSize) {\n chunks.push(jsonArray.slice(i, i + chunkSize));\n }\n return chunks.map(chunk => ({ json: { chunk } }));\n} catch (error) {\n return [{ json: { error: error.message } }];\n}\n"
},
"typeVersion": 2
},
{
"id": "5a3ac393-8355-449f-93cb-b98e8bee9b80",
"name": "Perform SerpAPI Search Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
-780,
180
],
"parameters": {
"url": "https://serpapi.com/search",
"options": {},
"sendQuery": true,
"queryParameters": {
"parameters": [
{
"name": "q",
"value": "={{ $('Parse and Chunk JSON Data').item.json.chunk }}"
},
{
"name": "api_key",
"value": "={{ $credentials.SerpAPI.key }}"
},
{
"name": "engine",
"value": "google"
}
]
}
},
"typeVersion": 4.2
},
{
"id": "dad82469-830d-40fb-9f6b-b9fefef41267",
"name": "Perform Jina AI Analysis Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
80,
160
],
"parameters": {
"url": "=https://r.jina.ai/{{ $json.url }}",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "iseKF5sPsvwtJhgT",
"name": "Jina AI"
}
},
"typeVersion": 4.2
},
{
"id": "e21bbdf6-a903-491e-920c-ef7576f9ce80",
"name": "Format SerpAPI Organic Results",
"type": "n8n-nodes-base.code",
"position": [
-460,
140
],
"parameters": {
"jsCode": "// Format the organic search results from SerpAPI\nconst results = $input.first().json.organic_results;\nif (results.length === 0) {\n return [{ json: { error: 'No search results found.' } }];\n}\nconst formattedResults = results.map(result => ({\n title: result.title || 'No title available',\n url: result.link || 'No link available',\n source: result.source || result.displayed_link || 'Unknown source'\n}));\nreturn formattedResults.map(result => ({ json: result }));\n"
},
"typeVersion": 2
},
{
"id": "a856c8e8-5c3c-4a2f-9086-66deee1afd06",
"name": "Extract Relevant Context via LLM",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
-1280,
520
],
"parameters": {
"text": "=User Queries: {{ $('Parse and Chunk JSON Data').all().map(item => item.json.chunk[0]).join(', ') }}\nWebpage Contents: \n\"\"\"\n{{ $json.data }}\n\"\"\"",
"options": {
"systemMessage": "=You are an expert information extractor. Given the user's query, the search query that led to this page, and the webpage content, extract all relevant pieces of information that are useful to answer the query. Return only the relevant context as plain text without any additional commentary."
},
"promptType": "define"
},
"typeVersion": 1.7
},
{
"id": "6d5c6698-0b4f-438c-91b9-3597f5d3e904",
"name": "Generate Comprehensive Research Report",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
-740,
520
],
"parameters": {
"text": "=Extracted Contexts (Merged):\n\"\"\"\n{{ $json.output }}\n\"\"\"",
"options": {
"systemMessage": "You are an expert researcher and report writer. Based on the gathered contexts and the original user query, generate a comprehensive, well-structured report. Include all relevant insights and conclusions without unnecessary commentary.\n\nFormat the report in Markdown with clear headings. For example:\n\n# Research Report: [User Query]\n\n## Key Findings\n- Point 1\n- Point 2\n\n## Detailed Analysis\n### Aspect 1\nSummary of findings.\n_Source:_ [Source Name](URL)\n\n### Aspect 2\nSummary of findings.\n_Source:_ [Another Source](URL)\n\nNow, generate the complete report."
},
"promptType": "define"
},
"typeVersion": 1.7
},
{
"id": "05fea6a1-791e-4980-8f2a-2960455066d7",
"name": "Split Data for SerpAPI Batching",
"type": "n8n-nodes-base.splitInBatches",
"position": [
-1100,
160
],
"parameters": {
"options": {}
},
"typeVersion": 3
},
{
"id": "df00e7e8-99b8-484a-8047-869474fefee9",
"name": "Split Data for Jina AI Batching",
"type": "n8n-nodes-base.splitInBatches",
"position": [
-220,
140
],
"parameters": {
"options": {}
},
"typeVersion": 3
},
{
"id": "2edc683b-65f7-40c3-a22d-7fbf5b67de0a",
"name": "LLM Memory Buffer (Input Context)",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
-1160,
740
],
"parameters": {
"sessionKey": "my_test_session",
"sessionIdType": "customKey",
"contextWindowLength": 20
},
"typeVersion": 1.3
},
{
"id": "23017ae7-72a7-45c7-8edf-d0ba72220675",
"name": "LLM Memory Buffer (Report Context)",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
-620,
760
],
"parameters": {
"sessionKey": "my_test_session",
"sessionIdType": "customKey",
"contextWindowLength": 20
},
"typeVersion": 1.3
},
{
"id": "6bc9533b-e265-47b3-b93a-3a4f86ba0541",
"name": "Fetch Wikipedia Information",
"type": "@n8n/n8n-nodes-langchain.toolWikipedia",
"position": [
-580,
920
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b25c148e-047d-40a7-8818-94c3504828dd",
"name": "Sticky Note: SerpAPI Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-20
],
"parameters": {
"color": 7,
"width": 420,
"height": 140,
"content": "## SerpAPI Setup Instructions\n1. Obtain your API key from https://serpapi.com/manage-api-key.\n2. Save your API key securely in n8n credentials (do not use plain text)."
},
"typeVersion": 1
},
{
"id": "e69c9a85-31e4-42b9-a09a-683ec5bb97d1",
"name": "Sticky Note: Jina AI Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-60,
-40
],
"parameters": {
"color": 7,
"width": 420,
"height": 140,
"content": "## Jina AI Setup Instructions\n1. Obtain your API key from https://jina.ai/api-dashboard/key-manager.\n2. Configure your Jina AI credential in n8n to ensure secure API access."
},
"typeVersion": 1
},
{
"id": "dbd204e0-da8e-41d8-814b-f409a23e9573",
"name": "Sticky Note: OpenRouter API Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1680,
460
],
"parameters": {
"color": 7,
"width": 300,
"height": 180,
"content": "## OpenRouter API Setup Instructions\n1. Obtain your API key from https://openrouter.ai/settings/keys.\n2. Set up your OpenRouter credential in n8n for secure integration."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "aa857bb3-84c1-4fe6-9464-90fc09163960",
"connections": {
"Chat Message Trigger": {
"main": [
[
{
"node": "Generate Search Queries using LLM",
"type": "main",
"index": 0
}
]
]
},
"Parse and Chunk JSON Data": {
"main": [
[
{
"node": "Split Data for SerpAPI Batching",
"type": "main",
"index": 0
}
]
]
},
"Fetch Wikipedia Information": {
"ai_tool": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "ai_tool",
"index": 0
}
]
]
},
"Format SerpAPI Organic Results": {
"main": [
[
{
"node": "Split Data for Jina AI Batching",
"type": "main",
"index": 0
}
]
]
},
"Perform SerpAPI Search Request": {
"main": [
[
{
"node": "Split Data for SerpAPI Batching",
"type": "main",
"index": 0
}
]
]
},
"Split Data for Jina AI Batching": {
"main": [
[
{
"node": "Extract Relevant Context via LLM",
"type": "main",
"index": 0
}
],
[
{
"node": "Perform Jina AI Analysis Request",
"type": "main",
"index": 0
}
]
]
},
"Split Data for SerpAPI Batching": {
"main": [
[
{
"node": "Format SerpAPI Organic Results",
"type": "main",
"index": 0
}
],
[
{
"node": "Perform SerpAPI Search Request",
"type": "main",
"index": 0
}
]
]
},
"Extract Relevant Context via LLM": {
"main": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "main",
"index": 0
}
]
]
},
"Perform Jina AI Analysis Request": {
"main": [
[
{
"node": "Split Data for Jina AI Batching",
"type": "main",
"index": 0
}
]
]
},
"Generate Search Queries using LLM": {
"main": [
[
{
"node": "Parse and Chunk JSON Data",
"type": "main",
"index": 0
}
]
]
},
"LLM Memory Buffer (Input Context)": {
"ai_memory": [
[
{
"node": "Extract Relevant Context via LLM",
"type": "ai_memory",
"index": 0
}
]
]
},
"LLM Memory Buffer (Report Context)": {
"ai_memory": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "ai_memory",
"index": 0
}
]
]
},
"LLM Response Provider (OpenRouter)": {
"ai_languageModel": [
[
{
"node": "Generate Search Queries using LLM",
"type": "ai_languageModel",
"index": 0
},
{
"node": "Extract Relevant Context via LLM",
"type": "ai_languageModel",
"index": 0
},
{
"node": "Generate Comprehensive Research Report",
"type": "ai_languageModel",
"index": 0
}
]
]
}
}
}

468
AI_Research_RAG_and_Data_Analysis/Open Deep Research - AI-Powered Autonomous Research Workflow.txt
View File

@@ -1,468 +0,0 @@
{
"id": "WLSqXECfQF7rOj2A",
"meta": {
"instanceId": "cba4a4a2eb5d7683330e2944837278938831ed3c042e20da6f5049c07ad14798"
},
"name": "Open Deep Research - AI-Powered Autonomous Research Workflow",
"tags": [],
"nodes": [
{
"id": "b7b70ba1-0267-4d2b-91f4-5cc4fd22fd03",
"name": "Chat Message Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
-1940,
160
],
"webhookId": "cb0b9dbe-1f35-441a-b062-29624b0ebc6a",
"parameters": {
"options": {}
},
"typeVersion": 1.1
},
{
"id": "55a8a512-f2d4-4aed-93e5-dd9bfa2dcaad",
"name": "Generate Search Queries using LLM",
"type": "@n8n/n8n-nodes-langchain.chainLlm",
"position": [
-1760,
160
],
"parameters": {
"text": "=User Query: {{ $('Chat Message Trigger').item.json.chatInput }}",
"messages": {
"messageValues": [
{
"message": "=You are an expert research assistant. Given a user's query, generate up to four distinct, precise search queries that would help gather comprehensive information on the topic. Return only a JSON list of strings, for example: ['query1', 'query2', 'query3']."
}
]
},
"promptType": "define"
},
"typeVersion": 1.5
},
{
"id": "5f92361a-b490-479d-8360-c87a100b470e",
"name": "LLM Response Provider (OpenRouter)",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter",
"position": [
-1760,
700
],
"parameters": {
"model": "google/gemini-2.0-flash-001",
"options": {}
},
"credentials": {
"openRouterApi": {
"id": "WZWYWCfluxuKxZzV",
"name": "OpenRouter account"
}
},
"typeVersion": 1
},
{
"id": "4ab360eb-858f-48b8-a00d-71867d4f0c93",
"name": "Parse and Chunk JSON Data",
"type": "n8n-nodes-base.code",
"position": [
-1420,
160
],
"parameters": {
"jsCode": "// Parse the input JSON string and split it into four chunks\nconst rawText = $json.text;\n\n// Remove Markdown JSON code blocks if present\nconst cleanedText = rawText.replace(/```json|```/g, '').trim();\n\ntry {\n const jsonArray = JSON.parse(cleanedText);\n if (!Array.isArray(jsonArray)) {\n throw new Error('The JSON is not an array.');\n }\n const chunkSize = Math.ceil(jsonArray.length / 4);\n const chunks = [];\n for (let i = 0; i < jsonArray.length; i += chunkSize) {\n chunks.push(jsonArray.slice(i, i + chunkSize));\n }\n return chunks.map(chunk => ({ json: { chunk } }));\n} catch (error) {\n return [{ json: { error: error.message } }];\n}\n"
},
"typeVersion": 2
},
{
"id": "5a3ac393-8355-449f-93cb-b98e8bee9b80",
"name": "Perform SerpAPI Search Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
-780,
180
],
"parameters": {
"url": "https://serpapi.com/search",
"options": {},
"sendQuery": true,
"queryParameters": {
"parameters": [
{
"name": "q",
"value": "={{ $('Parse and Chunk JSON Data').item.json.chunk }}"
},
{
"name": "api_key",
"value": "={{ $credentials.SerpAPI.key }}"
},
{
"name": "engine",
"value": "google"
}
]
}
},
"typeVersion": 4.2
},
{
"id": "dad82469-830d-40fb-9f6b-b9fefef41267",
"name": "Perform Jina AI Analysis Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
80,
160
],
"parameters": {
"url": "=https://r.jina.ai/{{ $json.url }}",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "iseKF5sPsvwtJhgT",
"name": "Jina AI"
}
},
"typeVersion": 4.2
},
{
"id": "e21bbdf6-a903-491e-920c-ef7576f9ce80",
"name": "Format SerpAPI Organic Results",
"type": "n8n-nodes-base.code",
"position": [
-460,
140
],
"parameters": {
"jsCode": "// Format the organic search results from SerpAPI\nconst results = $input.first().json.organic_results;\nif (results.length === 0) {\n return [{ json: { error: 'No search results found.' } }];\n}\nconst formattedResults = results.map(result => ({\n title: result.title || 'No title available',\n url: result.link || 'No link available',\n source: result.source || result.displayed_link || 'Unknown source'\n}));\nreturn formattedResults.map(result => ({ json: result }));\n"
},
"typeVersion": 2
},
{
"id": "a856c8e8-5c3c-4a2f-9086-66deee1afd06",
"name": "Extract Relevant Context via LLM",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
-1280,
520
],
"parameters": {
"text": "=User Queries: {{ $('Parse and Chunk JSON Data').all().map(item => item.json.chunk[0]).join(', ') }}\nWebpage Contents: \n\"\"\"\n{{ $json.data }}\n\"\"\"",
"options": {
"systemMessage": "=You are an expert information extractor. Given the user's query, the search query that led to this page, and the webpage content, extract all relevant pieces of information that are useful to answer the query. Return only the relevant context as plain text without any additional commentary."
},
"promptType": "define"
},
"typeVersion": 1.7
},
{
"id": "6d5c6698-0b4f-438c-91b9-3597f5d3e904",
"name": "Generate Comprehensive Research Report",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
-740,
520
],
"parameters": {
"text": "=Extracted Contexts (Merged):\n\"\"\"\n{{ $json.output }}\n\"\"\"",
"options": {
"systemMessage": "You are an expert researcher and report writer. Based on the gathered contexts and the original user query, generate a comprehensive, well-structured report. Include all relevant insights and conclusions without unnecessary commentary.\n\nFormat the report in Markdown with clear headings. For example:\n\n# Research Report: [User Query]\n\n## Key Findings\n- Point 1\n- Point 2\n\n## Detailed Analysis\n### Aspect 1\nSummary of findings.\n_Source:_ [Source Name](URL)\n\n### Aspect 2\nSummary of findings.\n_Source:_ [Another Source](URL)\n\nNow, generate the complete report."
},
"promptType": "define"
},
"typeVersion": 1.7
},
{
"id": "05fea6a1-791e-4980-8f2a-2960455066d7",
"name": "Split Data for SerpAPI Batching",
"type": "n8n-nodes-base.splitInBatches",
"position": [
-1100,
160
],
"parameters": {
"options": {}
},
"typeVersion": 3
},
{
"id": "df00e7e8-99b8-484a-8047-869474fefee9",
"name": "Split Data for Jina AI Batching",
"type": "n8n-nodes-base.splitInBatches",
"position": [
-220,
140
],
"parameters": {
"options": {}
},
"typeVersion": 3
},
{
"id": "2edc683b-65f7-40c3-a22d-7fbf5b67de0a",
"name": "LLM Memory Buffer (Input Context)",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
-1160,
740
],
"parameters": {
"sessionKey": "my_test_session",
"sessionIdType": "customKey",
"contextWindowLength": 20
},
"typeVersion": 1.3
},
{
"id": "23017ae7-72a7-45c7-8edf-d0ba72220675",
"name": "LLM Memory Buffer (Report Context)",
"type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
"position": [
-620,
760
],
"parameters": {
"sessionKey": "my_test_session",
"sessionIdType": "customKey",
"contextWindowLength": 20
},
"typeVersion": 1.3
},
{
"id": "6bc9533b-e265-47b3-b93a-3a4f86ba0541",
"name": "Fetch Wikipedia Information",
"type": "@n8n/n8n-nodes-langchain.toolWikipedia",
"position": [
-580,
920
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b25c148e-047d-40a7-8818-94c3504828dd",
"name": "Sticky Note: SerpAPI Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-20
],
"parameters": {
"color": 7,
"width": 420,
"height": 140,
"content": "## SerpAPI Setup Instructions\n1. Obtain your API key from https://serpapi.com/manage-api-key.\n2. Save your API key securely in n8n credentials (do not use plain text)."
},
"typeVersion": 1
},
{
"id": "e69c9a85-31e4-42b9-a09a-683ec5bb97d1",
"name": "Sticky Note: Jina AI Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-60,
-40
],
"parameters": {
"color": 7,
"width": 420,
"height": 140,
"content": "## Jina AI Setup Instructions\n1. Obtain your API key from https://jina.ai/api-dashboard/key-manager.\n2. Configure your Jina AI credential in n8n to ensure secure API access."
},
"typeVersion": 1
},
{
"id": "dbd204e0-da8e-41d8-814b-f409a23e9573",
"name": "Sticky Note: OpenRouter API Setup",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1680,
460
],
"parameters": {
"color": 7,
"width": 300,
"height": 180,
"content": "## OpenRouter API Setup Instructions\n1. Obtain your API key from https://openrouter.ai/settings/keys.\n2. Set up your OpenRouter credential in n8n for secure integration."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "aa857bb3-84c1-4fe6-9464-90fc09163960",
"connections": {
"Chat Message Trigger": {
"main": [
[
{
"node": "Generate Search Queries using LLM",
"type": "main",
"index": 0
}
]
]
},
"Parse and Chunk JSON Data": {
"main": [
[
{
"node": "Split Data for SerpAPI Batching",
"type": "main",
"index": 0
}
]
]
},
"Fetch Wikipedia Information": {
"ai_tool": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "ai_tool",
"index": 0
}
]
]
},
"Format SerpAPI Organic Results": {
"main": [
[
{
"node": "Split Data for Jina AI Batching",
"type": "main",
"index": 0
}
]
]
},
"Perform SerpAPI Search Request": {
"main": [
[
{
"node": "Split Data for SerpAPI Batching",
"type": "main",
"index": 0
}
]
]
},
"Split Data for Jina AI Batching": {
"main": [
[
{
"node": "Extract Relevant Context via LLM",
"type": "main",
"index": 0
}
],
[
{
"node": "Perform Jina AI Analysis Request",
"type": "main",
"index": 0
}
]
]
},
"Split Data for SerpAPI Batching": {
"main": [
[
{
"node": "Format SerpAPI Organic Results",
"type": "main",
"index": 0
}
],
[
{
"node": "Perform SerpAPI Search Request",
"type": "main",
"index": 0
}
]
]
},
"Extract Relevant Context via LLM": {
"main": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "main",
"index": 0
}
]
]
},
"Perform Jina AI Analysis Request": {
"main": [
[
{
"node": "Split Data for Jina AI Batching",
"type": "main",
"index": 0
}
]
]
},
"Generate Search Queries using LLM": {
"main": [
[
{
"node": "Parse and Chunk JSON Data",
"type": "main",
"index": 0
}
]
]
},
"LLM Memory Buffer (Input Context)": {
"ai_memory": [
[
{
"node": "Extract Relevant Context via LLM",
"type": "ai_memory",
"index": 0
}
]
]
},
"LLM Memory Buffer (Report Context)": {
"ai_memory": [
[
{
"node": "Generate Comprehensive Research Report",
"type": "ai_memory",
"index": 0
}
]
]
},
"LLM Response Provider (OpenRouter)": {
"ai_languageModel": [
[
{
"node": "Generate Search Queries using LLM",
"type": "ai_languageModel",
"index": 0
},
{
"node": "Extract Relevant Context via LLM",
"type": "ai_languageModel",
"index": 0
},
{
"node": "Generate Comprehensive Research Report",
"type": "ai_languageModel",
"index": 0
}
]
]
}
}
}

176
AI_Research_RAG_and_Data_Analysis/Query Perplexity AI from your n8n workflows.json Normal file
View File

@@ -0,0 +1,176 @@
{
"nodes": [
{
"id": "293b70f0-06e8-4db5-befd-bfaed1f3575a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-460,
80
],
"parameters": {},
"typeVersion": 1
},
{
"id": "1c473546-6280-412d-9f8e-b43962365d78",
"name": "Set Params",
"type": "n8n-nodes-base.set",
"position": [
-160,
-60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "8b5c6ca0-5ca8-4f67-abc1-44341cf419bc",
"name": "system_prompt",
"type": "string",
"value": "You are an n8n fanboy."
},
{
"id": "7c36c362-6269-4564-b6fe-f82126bc8f5e",
"name": "user_prompt",
"type": "string",
"value": "What are the differences between n8n and Make?"
},
{
"id": "4366d2b5-ad22-445a-8589-fddab1caa1ab",
"name": "domains",
"type": "string",
"value": "n8n.io, make.com"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "894bd6a4-5db7-45fb-a8e0-1a81af068bbf",
"name": "Clean Output",
"type": "n8n-nodes-base.set",
"position": [
580,
-100
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5859093c-6b22-41db-ac6c-9a9f6f18b7e3",
"name": "output",
"type": "string",
"value": "={{ $json.choices[0].message.content }}"
},
{
"id": "13208fff-5153-45a7-a1cb-fe49e32d9a03",
"name": "citations",
"type": "array",
"value": "={{ $json.citations }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "52d3a832-8c9b-4356-ad2a-377340678a58",
"name": "Perplexity Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
40
],
"parameters": {
"url": "https://api.perplexity.ai/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"sonar\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{ $json.system_prompt }}\"\n },\n {\n \"role\": \"user\",\n \"content\": \"{{ $json.user_prompt }}\"\n }\n ],\n \"temperature\": 0.2,\n \"top_p\": 0.9,\n \"search_domain_filter\": {{ (JSON.stringify($json.domains.split(','))) }},\n \"return_images\": false,\n \"return_related_questions\": false,\n \"search_recency_filter\": \"month\",\n \"top_k\": 0,\n \"stream\": false,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 1,\n \"response_format\": null\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpBasicAuth": {
"id": "yEocL0NSpUWzMsHG",
"name": "Perplexity"
},
"httpHeaderAuth": {
"id": "TngzgS09J1YvLIXl",
"name": "Perplexity"
}
},
"typeVersion": 4.2
},
{
"id": "48657f2c-d1dd-4d7e-8014-c27748e63e58",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-140,
-440
],
"parameters": {
"width": 480,
"height": 300,
"content": "## Credentials Setup\n\n1/ Go to the perplexity dashboard, purchase some credits and create an API Key\n\nhttps://www.perplexity.ai/settings/api\n\n2/ In the perplexity Request node, use Generic Credentials, Header Auth. \n\nFor the name, use the value \"Authorization\"\nAnd for the value \"Bearer pplx-e4...59ea\" (Your Perplexity Api Key)\n\n"
},
"typeVersion": 1
},
{
"id": "e0daabee-c145-469e-93c2-c759c303dc2a",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
100,
260
],
"parameters": {
"color": 5,
"width": 480,
"height": 120,
"content": "**Sonar Pro** is the current top model used by perplexity. \nIf you want to use a different one, check this page: \n\nhttps://docs.perplexity.ai/guides/model-cards"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Set Params": {
"main": [
[
{
"node": "Perplexity Request",
"type": "main",
"index": 0
}
]
]
},
"Perplexity Request": {
"main": [
[
{
"node": "Clean Output",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Params",
"type": "main",
"index": 0
}
]
]
}
}
}

176
AI_Research_RAG_and_Data_Analysis/Query Perplexity AI from your n8n workflows.txt
View File

@@ -1,176 +0,0 @@
{
"nodes": [
{
"id": "293b70f0-06e8-4db5-befd-bfaed1f3575a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-460,
80
],
"parameters": {},
"typeVersion": 1
},
{
"id": "1c473546-6280-412d-9f8e-b43962365d78",
"name": "Set Params",
"type": "n8n-nodes-base.set",
"position": [
-160,
-60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "8b5c6ca0-5ca8-4f67-abc1-44341cf419bc",
"name": "system_prompt",
"type": "string",
"value": "You are an n8n fanboy."
},
{
"id": "7c36c362-6269-4564-b6fe-f82126bc8f5e",
"name": "user_prompt",
"type": "string",
"value": "What are the differences between n8n and Make?"
},
{
"id": "4366d2b5-ad22-445a-8589-fddab1caa1ab",
"name": "domains",
"type": "string",
"value": "n8n.io, make.com"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "894bd6a4-5db7-45fb-a8e0-1a81af068bbf",
"name": "Clean Output",
"type": "n8n-nodes-base.set",
"position": [
580,
-100
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5859093c-6b22-41db-ac6c-9a9f6f18b7e3",
"name": "output",
"type": "string",
"value": "={{ $json.choices[0].message.content }}"
},
{
"id": "13208fff-5153-45a7-a1cb-fe49e32d9a03",
"name": "citations",
"type": "array",
"value": "={{ $json.citations }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "52d3a832-8c9b-4356-ad2a-377340678a58",
"name": "Perplexity Request",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
40
],
"parameters": {
"url": "https://api.perplexity.ai/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"sonar\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{ $json.system_prompt }}\"\n },\n {\n \"role\": \"user\",\n \"content\": \"{{ $json.user_prompt }}\"\n }\n ],\n \"temperature\": 0.2,\n \"top_p\": 0.9,\n \"search_domain_filter\": {{ (JSON.stringify($json.domains.split(','))) }},\n \"return_images\": false,\n \"return_related_questions\": false,\n \"search_recency_filter\": \"month\",\n \"top_k\": 0,\n \"stream\": false,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 1,\n \"response_format\": null\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpBasicAuth": {
"id": "yEocL0NSpUWzMsHG",
"name": "Perplexity"
},
"httpHeaderAuth": {
"id": "TngzgS09J1YvLIXl",
"name": "Perplexity"
}
},
"typeVersion": 4.2
},
{
"id": "48657f2c-d1dd-4d7e-8014-c27748e63e58",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-140,
-440
],
"parameters": {
"width": 480,
"height": 300,
"content": "## Credentials Setup\n\n1/ Go to the perplexity dashboard, purchase some credits and create an API Key\n\nhttps://www.perplexity.ai/settings/api\n\n2/ In the perplexity Request node, use Generic Credentials, Header Auth. \n\nFor the name, use the value \"Authorization\"\nAnd for the value \"Bearer pplx-e4...59ea\" (Your Perplexity Api Key)\n\n"
},
"typeVersion": 1
},
{
"id": "e0daabee-c145-469e-93c2-c759c303dc2a",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
100,
260
],
"parameters": {
"color": 5,
"width": 480,
"height": 120,
"content": "**Sonar Pro** is the current top model used by perplexity. \nIf you want to use a different one, check this page: \n\nhttps://docs.perplexity.ai/guides/model-cards"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Set Params": {
"main": [
[
{
"node": "Perplexity Request",
"type": "main",
"index": 0
}
]
]
},
"Perplexity Request": {
"main": [
[
{
"node": "Clean Output",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Params",
"type": "main",
"index": 0
}
]
]
}
}
}

973
AI_Research_RAG_and_Data_Analysis/Recipe Recommendations with Qdrant and Mistral.json Normal file
View File

@@ -0,0 +1,973 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "1eb82902-a1d6-4eff-82a2-26908a82cea2",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
720,
320
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e0031fc3-27f1-45d9-910b-4c07dd322115",
"name": "Get This Week's Menu",
"type": "n8n-nodes-base.httpRequest",
"position": [
992,
370
],
"parameters": {
"url": "=https://www.hellofresh.co.uk/menus/{{ $now.year }}-W{{ $now.weekNumber }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "2c556cc7-7d4e-4d80-902f-9686e756ed8c",
"name": "Extract Available Courses",
"type": "n8n-nodes-base.code",
"position": [
992,
650
],
"parameters": {
"jsCode": "const pageData = JSON.parse($input.first().json.data)\nreturn pageData.props.pageProps.ssrPayload.courses.slice(0, 10);"
},
"typeVersion": 2
},
{
"id": "90c39db6-6116-4c37-8d48-a6d5e8f8c777",
"name": "Extract Server Data",
"type": "n8n-nodes-base.html",
"position": [
992,
510
],
"parameters": {
"options": {
"trimValues": false,
"cleanUpText": true
},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"cssSelector": "script#__NEXT_DATA__"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "fbd4ed97-0154-4991-bf16-d9c4cb3f4776",
"name": "Get Course Metadata",
"type": "n8n-nodes-base.set",
"position": [
1172,
370
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "3c90fd1e-e9ac-49c1-a459-7cff8c87fe8d",
"name": "name",
"type": "string",
"value": "={{ $json.recipe.name }}"
},
{
"id": "c4f3a5df-346c-4e8d-90ba-a49ed6afdedf",
"name": "cuisines",
"type": "array",
"value": "={{ $json.recipe.cuisines.map(item => item.name) }}"
},
{
"id": "97917928-0956-497b-bb68-507df1783240",
"name": "category",
"type": "string",
"value": "={{ $json.recipe.category.name }}"
},
{
"id": "1e84cf1e-7ad7-4888-9606-d3f7a310ce5f",
"name": "tags",
"type": "array",
"value": "={{ $json.recipe.tags.flatMap(tag => tag.preferences) }}"
},
{
"id": "cf6e2174-e8cb-4935-8303-2f8ed067f510",
"name": "nutrition",
"type": "object",
"value": "={{ $json.recipe.nutrition.reduce((acc,item) => ({ ...acc, [item.name]: item.amount + item.unit }), {}) }}"
},
{
"id": "25ba3fe6-c2fa-4315-a2cb-112ec7e3620f",
"name": "url",
"type": "string",
"value": "={{ $json.recipe.websiteUrl }}"
},
{
"id": "8f444fb3-c2ee-4254-b505-440cca3c7b8b",
"name": "id",
"type": "string",
"value": "={{ $json.recipe.id }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "5ab1a5fa-adc3-41e0-be6d-f680af301aca",
"name": "Get Recipe",
"type": "n8n-nodes-base.httpRequest",
"position": [
1172,
510
],
"parameters": {
"url": "={{ $json.recipe.websiteUrl }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "5014dc62-8320-4968-b9bd-396a517a2b5c",
"name": "Embeddings Mistral Cloud",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
1960,
420
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "2a8fad89-f74b-4808-8cb6-97c6b46a53ee",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
2080,
420
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "week",
"value": "={{ $json.week }}"
},
{
"name": "cuisine",
"value": "={{ $json.cuisines }}"
},
{
"name": "category",
"value": "={{ $json.category }}"
},
{
"name": "tag",
"value": "={{ $json.tags }}"
},
{
"name": "recipe_id",
"value": "={{ $json.id }}"
}
]
}
},
"jsonData": "={{ $json.data }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "44ceef5c-1d08-40d2-8ab4-227b551f72f5",
"name": "Merge Course & Recipe",
"type": "n8n-nodes-base.merge",
"position": [
1480,
500
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "b56bd85e-f182-49d1-aeb1-062e905c316a",
"name": "Prepare Documents",
"type": "n8n-nodes-base.set",
"position": [
1660,
500
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "462567fe-02ec-4747-ae33-407d2bc6d776",
"name": "data",
"type": "string",
"value": "=# {{ $json.name }}\n{{ $json.description.replaceAll('\\n\\n','\\n') }}\n\n# Website\n{{ $json.url }}\n\n## Ingredients\n{{ $json.ingredients.replaceAll('\\n\\n','\\n') }}\n\n## Utensils\n{{ $json.utensils }}\n\n## Nutrition\n{{ Object.keys($json.nutrition).map(key => `* ${key}: ${$json.nutrition[key]}`).join('\\n') }}\n\n## Instructions\n{{ $json.instructions.replaceAll('\\n\\n','\\n') }}"
},
{
"id": "5738e420-abfe-4a85-b7ad-541cfc181563",
"name": "cuisine",
"type": "array",
"value": "={{ $json.cuisines }}"
},
{
"id": "349f46d4-e230-4da8-a118-50227ceb7233",
"name": "category",
"type": "string",
"value": "={{ $json.category }}"
},
{
"id": "9588b347-4469-4aa5-93a2-e7bf41b4c468",
"name": "tag",
"type": "array",
"value": "={{ $json.tags }}"
},
{
"id": "7ddab229-fa52-4d27-84e1-83ed47280d29",
"name": "week",
"type": "string",
"value": "={{ $now.year }}-W{{ $now.weekNumber }}"
},
{
"id": "13163e45-5699-4d25-af3d-4c7910dd2926",
"name": "id",
"type": "string",
"value": "={{ $json.id }}"
},
{
"id": "a0c5d599-ff2b-420d-9173-2baf9218abc5",
"name": "name",
"type": "string",
"value": "={{ $json.name }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "6b800632-f320-4fc3-bd2a-6a062834343d",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
2080,
560
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "df7f17a2-8b27-4203-a2ff-091aaf6609b8",
"name": "Chat Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
2440,
360
],
"webhookId": "745056ec-2d36-4ac3-9c70-6ff0b1055d0a",
"parameters": {},
"typeVersion": 1
},
{
"id": "ee38effe-5929-421e-a3c5-b1055a755242",
"name": "Extract Recipe Details",
"type": "n8n-nodes-base.html",
"position": [
1172,
650
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "description",
"cssSelector": "[data-test-id=\"recipe-description\"]"
},
{
"key": "ingredients",
"cssSelector": "[data-test-id=\"ingredients-list\"]"
},
{
"key": "utensils",
"cssSelector": "[data-test-id=\"utensils\"]"
},
{
"key": "instructions",
"cssSelector": "[data-test-id=\"instructions\"]",
"skipSelectors": "img,a"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "dede108f-2fde-49cb-8a0e-fa5786c59d4b",
"name": "Qdrant Recommend API",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
2840,
540
],
"parameters": {
"name": "get_recipe_recommendation",
"fields": {
"values": [
{
"name": "week",
"stringValue": "={{ $now.year }}-W{{ $now.weekNumber }}"
}
]
},
"schemaType": "manual",
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to get a recipe recommendation. Pass in the following params as a json object:\n* positives - a description of what the user wants to cook. This could be ingredients, flavours, utensils available, number of diners, type of meal etc.\n* negatives - a description of what the user wants to avoid in the recipe. This could be flavours to avoid, allergen considerations, conflicts with theme of meal etc.",
"inputSchema": "{\n\"type\": \"object\",\n\"properties\": {\n\t\"positive\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"a description of what the user wants to cook. This could be ingredients, flavours, utensils available, number of diners, type of meal etc.\"\n\t},\n \"negative\": {\n \"type\": \"string\",\n \"description\": \"a description of what the user wants to avoid in the recipe. This could be flavours to avoid, allergen considerations, conflicts with theme of meal etc.\"\n }\n}\n}",
"specifyInputSchema": true
},
"typeVersion": 1.1
},
{
"id": "5e703134-4dd9-464b-9ec9-dc6103907a1e",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
2420,
940
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9fb5f4fd-3b38-4a35-8986-d3955754c8d1",
"name": "Mistral Cloud Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatMistralCloud",
"position": [
2660,
540
],
"parameters": {
"model": "mistral-large-2402",
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "d38275e6-aede-4f1c-9b05-018f3cf4faab",
"name": "Get Tool Response",
"type": "n8n-nodes-base.set",
"position": [
3160,
940
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "10b55200-4610-4e9b-8be7-d487c6b56a78",
"name": "response",
"type": "string",
"value": "={{ JSON.stringify($json.result) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "dc3ceb2f-3c64-4b42-aeca-ddcdb84abf12",
"name": "Wait for Rate Limits",
"type": "n8n-nodes-base.wait",
"position": [
2420,
1080
],
"webhookId": "e86d8ae4-3b0d-4c40-9d12-a11d6501a043",
"parameters": {
"amount": 1.1
},
"typeVersion": 1.1
},
{
"id": "ec36d6f8-c3da-4732-8d56-a092a3358864",
"name": "Get Mistral Embeddings",
"type": "n8n-nodes-base.httpRequest",
"position": [
2620,
940
],
"parameters": {
"url": "https://api.mistral.ai/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "model",
"value": "mistral-embed"
},
{
"name": "encoding_format",
"value": "float"
},
{
"name": "input",
"value": "={{ [$json.query.positive, $json.query.negative].compact() }}"
}
]
},
"nodeCredentialType": "mistralCloudApi"
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 4.2
},
{
"id": "aebcb860-d25c-4833-9e9d-0297101259c7",
"name": "Use Qdrant Recommend API",
"type": "n8n-nodes-base.httpRequest",
"position": [
2800,
940
],
"parameters": {
"url": "=http://qdrant:6333/collections/hello_fresh/points/recommend/groups",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "strategy",
"value": "average_vector"
},
{
"name": "limit",
"value": "={{ 3 }}"
},
{
"name": "positive",
"value": "={{ [$json.data[0].embedding] }}"
},
{
"name": "negative",
"value": "={{ [$json.data[1].embedding] }}"
},
{
"name": "filter",
"value": "={{ { \"must\": {\"key\": \"metadata.week\", \"match\": { \"value\": $('Execute Workflow Trigger').item.json.week } } } }}"
},
{
"name": "with_payload",
"value": "={{ true }}"
},
{
"name": "group_by",
"value": "metadata.recipe_id"
},
{
"name": "group_size",
"value": "={{ 3 }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "2474c97d-0d85-4acc-a95e-2eb6494786dc",
"name": "Get Recipes From DB",
"type": "n8n-nodes-base.code",
"position": [
2980,
940
],
"parameters": {
"language": "python",
"pythonCode": "import sqlite3\ncon = sqlite3.connect(\"hello_fresh_1.db\")\n\nrecipe_ids = list(set([group.id for group in _input.all()[0].json.result.groups if group.hits[0].score > 0.5]))\nplaceholders = ','.join(['?' for i in range(0,len(recipe_ids))])\n\ncur = con.cursor()\nres = cur.execute(f'SELECT * FROM recipes WHERE id IN ({placeholders})', recipe_ids)\nrows = res.fetchall()\n\ncon.close()\n\nreturn [{ \"result\": [row[2] for row in rows] }]"
},
"typeVersion": 2
},
{
"id": "54229c2a-6e26-4350-8a94-57f415ef2340",
"name": "Save Recipes to DB",
"type": "n8n-nodes-base.code",
"position": [
1960,
940
],
"parameters": {
"language": "python",
"pythonCode": "import sqlite3\ncon = sqlite3.connect(\"hello_fresh_1.db\")\n\ncur = con.cursor()\ncur.execute(\"CREATE TABLE IF NOT EXISTS recipes (id TEXT PRIMARY KEY, name TEXT, data TEXT, cuisine TEXT, category TEXT, tag TEXT, week TEXT);\")\n\nfor item in _input.all():\n cur.execute('INSERT OR REPLACE INTO recipes VALUES(?,?,?,?,?,?,?)', (\n item.json.id,\n item.json.name,\n item.json.data,\n ','.join(item.json.cuisine),\n item.json.category,\n ','.join(item.json.tag),\n item.json.week\n ))\n\ncon.commit()\ncon.close()\n\nreturn [{ \"affected_rows\": len(_input.all()) }]"
},
"typeVersion": 2
},
{
"id": "725c1f56-5373-4891-92b9-3f32dd28892b",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
901.1666225087287,
180.99920515712074
],
"parameters": {
"color": 7,
"width": 484.12381677448207,
"height": 674.1153489831718,
"content": "## Step 1. Fetch Available Courses For the Current Week\n\nTo populate our vectorstore, we'll scrape the weekly menu off the HelloFresh Website. The pages are quite large so may take a while so please be patient."
},
"typeVersion": 1
},
{
"id": "f4e882b8-3762-4e6b-9e95-b0d708d0c284",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1420,
300
],
"parameters": {
"color": 7,
"width": 409.1756468632768,
"height": 398.81415970574335,
"content": "## Step 2. Create Recipe Documents For VectorStore\n\nTo populate our vectorstore, we'll scrape the weekly menu off the HelloFresh Website. The pages are quite large so may take a while so please be patient."
},
"typeVersion": 1
},
{
"id": "fc3c2221-b67c-451c-9096-d6acd2a297fa",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
19.326425127730317
],
"parameters": {
"color": 7,
"width": 486.02284096214964,
"height": 690.7816167755491,
"content": "## Step 3. Vectorise Recipes For Recommendation Engine\n[Read more about Qdrant node](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreqdrant)\n\nWe'll store our documents in our Qdrant vectorstore by converting to vectors using Mistral Embed. Our goal is to a build a recommendation engine for meals of the week which Qdrant is a perfect solution."
},
"typeVersion": 1
},
{
"id": "43296173-b929-46cc-b6ea-58007837b8df",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1740,
740
],
"parameters": {
"color": 7,
"width": 547.0098868353456,
"height": 347.6002738958705,
"content": "## Step 4. Save Original Document to Database\n[Read more about Code Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.code)\n\nFinally, let's have the original document stored in a more traditional datastore. USually our vectorsearch will return partial docs and those are enough for many use-cases, however in this instance we'll pull the full docs for the Agent get the info required to make the recommendation. "
},
"typeVersion": 1
},
{
"id": "6e2e58d2-e0ad-4503-8ed6-891124c8035b",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2380,
160
],
"parameters": {
"color": 7,
"width": 673.6008766895472,
"height": 552.9202706743265,
"content": "## 5. Chat with Our HelloFresh Recommendation AI Agent\n[Read more about AI Agents](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent)\n\nThis agent is designed to recommend HelloFresh recipes based on your current taste preferences. Need something hot and spicy, warm and comforting or fast and chilled? This agent will capture what you would like and not like and queries our Recipe Recommendation engine powered by Qdrant Vectorstore."
},
"typeVersion": 1
},
{
"id": "ba692c21-38bc-48a1-8b40-bad298be8b9e",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
2660,
360
],
"parameters": {
"options": {
"systemMessage": "=You are a recipe bot for the company, \"Hello fresh\". You will help the user choose which Hello Fresh recipe to choose from this week's menu. The current week is {{ $now.year }}-W{{ $now.weekNumber }}.\nDo not recommend any recipes other from the current week's menu. If there are no recipes to recommend, please ask the user to visit the website instead https://hellofresh.com."
}
},
"typeVersion": 1.6
},
{
"id": "d7ca0f97-72dc-4f4c-8b46-3ff57b9068a4",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2320,
740
],
"parameters": {
"color": 7,
"width": 987.4785537889618,
"height": 531.9173034334732,
"content": "## 5. Using Qdrant's Recommend API & Grouping Functionality\n[Read more about Qdrant's Recommend API](https://qdrant.tech/documentation/concepts/explore/?q=recommend)\n\nUnlike basic similarity search, Qdrant's Recommend API takes a positive query to match against (eg. Roast Dinner) and a negative query to avoid (eg. Roast Chicken). This feature significantly improves results for a recommendation engine. Additionally, by utilising Qdrant's Grouping feature, we're able to group similar matches from the same recipe - meaning we can ensure unique recipes everytime."
},
"typeVersion": 1
},
{
"id": "96a294e2-1437-4ded-9973-0999b444c999",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-40
],
"parameters": {
"width": 432.916474478624,
"height": 542.9295980774649,
"content": "## Try it out!\n### This workflow does the following:\n* Fetches and stores this week's HelloFresh's menu\n* Builds the foundation of a recommendation engine by storing the recipes in a Qdrant Vectorstore and SQLite database.\n* Builds an AI Agent that allows for a chat interface to query for a the week's recipe recommendations.\n* AI agent uses the Qdrant Recommend API, providing what the user likes/dislikes as the query.\n* Qdrant returns the results which enable the AI Agent to make the recommendation to the user.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "72c98600-f21a-42d4-97be-836b8ef6dc77",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
1960,
240
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "list",
"value": "hello_fresh",
"cachedResultName": "hello_fresh"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
},
{
"id": "b7c4b597-ac2b-41d7-8f0f-1cbba25085de",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
-195.8987124522777
],
"parameters": {
"width": 382.47301504497716,
"height": 195.8987124522777,
"content": "### 🚨Ensure Qdrant collection exists!\nYou'll need to run the following command in Qdrant:\n```\nPUT collections/hello_fresh\n{\n \"vectors\": {\n \"distance\": \"Cosine\",\n \"size\": 1024\n }\n}\n```"
},
"typeVersion": 1
},
{
"id": "39191834-ecc2-46f0-a31a-0a7e9c47ac5d",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
2740,
920
],
"parameters": {
"width": 213.30551928619226,
"height": 332.38559808882246,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n### 🚨Configure Your Qdrant Connection\n* Be sure to enter your endpoint address"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Get Recipe": {
"main": [
[
{
"node": "Extract Recipe Details",
"type": "main",
"index": 0
}
]
]
},
"Chat Trigger": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Prepare Documents": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
},
{
"node": "Save Recipes to DB",
"type": "main",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Extract Server Data": {
"main": [
[
{
"node": "Extract Available Courses",
"type": "main",
"index": 0
}
]
]
},
"Get Course Metadata": {
"main": [
[
{
"node": "Merge Course & Recipe",
"type": "main",
"index": 0
}
]
]
},
"Get Recipes From DB": {
"main": [
[
{
"node": "Get Tool Response",
"type": "main",
"index": 0
}
]
]
},
"Get This Week's Menu": {
"main": [
[
{
"node": "Extract Server Data",
"type": "main",
"index": 0
}
]
]
},
"Qdrant Recommend API": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Wait for Rate Limits": {
"main": [
[
{
"node": "Get Mistral Embeddings",
"type": "main",
"index": 0
}
]
]
},
"Merge Course & Recipe": {
"main": [
[
{
"node": "Prepare Documents",
"type": "main",
"index": 0
}
]
]
},
"Extract Recipe Details": {
"main": [
[
{
"node": "Merge Course & Recipe",
"type": "main",
"index": 1
}
]
]
},
"Get Mistral Embeddings": {
"main": [
[
{
"node": "Use Qdrant Recommend API",
"type": "main",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Wait for Rate Limits",
"type": "main",
"index": 0
}
]
]
},
"Mistral Cloud Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Use Qdrant Recommend API": {
"main": [
[
{
"node": "Get Recipes From DB",
"type": "main",
"index": 0
}
]
]
},
"Extract Available Courses": {
"main": [
[
{
"node": "Get Course Metadata",
"type": "main",
"index": 0
},
{
"node": "Get Recipe",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Get This Week's Menu",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}

973
AI_Research_RAG_and_Data_Analysis/Recipe Recommendations with Qdrant and Mistral.txt
View File

@@ -1,973 +0,0 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "1eb82902-a1d6-4eff-82a2-26908a82cea2",
"name": "When clicking \"Test workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
720,
320
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e0031fc3-27f1-45d9-910b-4c07dd322115",
"name": "Get This Week's Menu",
"type": "n8n-nodes-base.httpRequest",
"position": [
992,
370
],
"parameters": {
"url": "=https://www.hellofresh.co.uk/menus/{{ $now.year }}-W{{ $now.weekNumber }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "2c556cc7-7d4e-4d80-902f-9686e756ed8c",
"name": "Extract Available Courses",
"type": "n8n-nodes-base.code",
"position": [
992,
650
],
"parameters": {
"jsCode": "const pageData = JSON.parse($input.first().json.data)\nreturn pageData.props.pageProps.ssrPayload.courses.slice(0, 10);"
},
"typeVersion": 2
},
{
"id": "90c39db6-6116-4c37-8d48-a6d5e8f8c777",
"name": "Extract Server Data",
"type": "n8n-nodes-base.html",
"position": [
992,
510
],
"parameters": {
"options": {
"trimValues": false,
"cleanUpText": true
},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"cssSelector": "script#__NEXT_DATA__"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "fbd4ed97-0154-4991-bf16-d9c4cb3f4776",
"name": "Get Course Metadata",
"type": "n8n-nodes-base.set",
"position": [
1172,
370
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "3c90fd1e-e9ac-49c1-a459-7cff8c87fe8d",
"name": "name",
"type": "string",
"value": "={{ $json.recipe.name }}"
},
{
"id": "c4f3a5df-346c-4e8d-90ba-a49ed6afdedf",
"name": "cuisines",
"type": "array",
"value": "={{ $json.recipe.cuisines.map(item => item.name) }}"
},
{
"id": "97917928-0956-497b-bb68-507df1783240",
"name": "category",
"type": "string",
"value": "={{ $json.recipe.category.name }}"
},
{
"id": "1e84cf1e-7ad7-4888-9606-d3f7a310ce5f",
"name": "tags",
"type": "array",
"value": "={{ $json.recipe.tags.flatMap(tag => tag.preferences) }}"
},
{
"id": "cf6e2174-e8cb-4935-8303-2f8ed067f510",
"name": "nutrition",
"type": "object",
"value": "={{ $json.recipe.nutrition.reduce((acc,item) => ({ ...acc, [item.name]: item.amount + item.unit }), {}) }}"
},
{
"id": "25ba3fe6-c2fa-4315-a2cb-112ec7e3620f",
"name": "url",
"type": "string",
"value": "={{ $json.recipe.websiteUrl }}"
},
{
"id": "8f444fb3-c2ee-4254-b505-440cca3c7b8b",
"name": "id",
"type": "string",
"value": "={{ $json.recipe.id }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "5ab1a5fa-adc3-41e0-be6d-f680af301aca",
"name": "Get Recipe",
"type": "n8n-nodes-base.httpRequest",
"position": [
1172,
510
],
"parameters": {
"url": "={{ $json.recipe.websiteUrl }}",
"options": {}
},
"typeVersion": 4.2
},
{
"id": "5014dc62-8320-4968-b9bd-396a517a2b5c",
"name": "Embeddings Mistral Cloud",
"type": "@n8n/n8n-nodes-langchain.embeddingsMistralCloud",
"position": [
1960,
420
],
"parameters": {
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "2a8fad89-f74b-4808-8cb6-97c6b46a53ee",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
2080,
420
],
"parameters": {
"options": {
"metadata": {
"metadataValues": [
{
"name": "week",
"value": "={{ $json.week }}"
},
{
"name": "cuisine",
"value": "={{ $json.cuisines }}"
},
{
"name": "category",
"value": "={{ $json.category }}"
},
{
"name": "tag",
"value": "={{ $json.tags }}"
},
{
"name": "recipe_id",
"value": "={{ $json.id }}"
}
]
}
},
"jsonData": "={{ $json.data }}",
"jsonMode": "expressionData"
},
"typeVersion": 1
},
{
"id": "44ceef5c-1d08-40d2-8ab4-227b551f72f5",
"name": "Merge Course & Recipe",
"type": "n8n-nodes-base.merge",
"position": [
1480,
500
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "b56bd85e-f182-49d1-aeb1-062e905c316a",
"name": "Prepare Documents",
"type": "n8n-nodes-base.set",
"position": [
1660,
500
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "462567fe-02ec-4747-ae33-407d2bc6d776",
"name": "data",
"type": "string",
"value": "=# {{ $json.name }}\n{{ $json.description.replaceAll('\\n\\n','\\n') }}\n\n# Website\n{{ $json.url }}\n\n## Ingredients\n{{ $json.ingredients.replaceAll('\\n\\n','\\n') }}\n\n## Utensils\n{{ $json.utensils }}\n\n## Nutrition\n{{ Object.keys($json.nutrition).map(key => `* ${key}: ${$json.nutrition[key]}`).join('\\n') }}\n\n## Instructions\n{{ $json.instructions.replaceAll('\\n\\n','\\n') }}"
},
{
"id": "5738e420-abfe-4a85-b7ad-541cfc181563",
"name": "cuisine",
"type": "array",
"value": "={{ $json.cuisines }}"
},
{
"id": "349f46d4-e230-4da8-a118-50227ceb7233",
"name": "category",
"type": "string",
"value": "={{ $json.category }}"
},
{
"id": "9588b347-4469-4aa5-93a2-e7bf41b4c468",
"name": "tag",
"type": "array",
"value": "={{ $json.tags }}"
},
{
"id": "7ddab229-fa52-4d27-84e1-83ed47280d29",
"name": "week",
"type": "string",
"value": "={{ $now.year }}-W{{ $now.weekNumber }}"
},
{
"id": "13163e45-5699-4d25-af3d-4c7910dd2926",
"name": "id",
"type": "string",
"value": "={{ $json.id }}"
},
{
"id": "a0c5d599-ff2b-420d-9173-2baf9218abc5",
"name": "name",
"type": "string",
"value": "={{ $json.name }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "6b800632-f320-4fc3-bd2a-6a062834343d",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
2080,
560
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "df7f17a2-8b27-4203-a2ff-091aaf6609b8",
"name": "Chat Trigger",
"type": "@n8n/n8n-nodes-langchain.chatTrigger",
"position": [
2440,
360
],
"webhookId": "745056ec-2d36-4ac3-9c70-6ff0b1055d0a",
"parameters": {},
"typeVersion": 1
},
{
"id": "ee38effe-5929-421e-a3c5-b1055a755242",
"name": "Extract Recipe Details",
"type": "n8n-nodes-base.html",
"position": [
1172,
650
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "description",
"cssSelector": "[data-test-id=\"recipe-description\"]"
},
{
"key": "ingredients",
"cssSelector": "[data-test-id=\"ingredients-list\"]"
},
{
"key": "utensils",
"cssSelector": "[data-test-id=\"utensils\"]"
},
{
"key": "instructions",
"cssSelector": "[data-test-id=\"instructions\"]",
"skipSelectors": "img,a"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "dede108f-2fde-49cb-8a0e-fa5786c59d4b",
"name": "Qdrant Recommend API",
"type": "@n8n/n8n-nodes-langchain.toolWorkflow",
"position": [
2840,
540
],
"parameters": {
"name": "get_recipe_recommendation",
"fields": {
"values": [
{
"name": "week",
"stringValue": "={{ $now.year }}-W{{ $now.weekNumber }}"
}
]
},
"schemaType": "manual",
"workflowId": "={{ $workflow.id }}",
"description": "Call this tool to get a recipe recommendation. Pass in the following params as a json object:\n* positives - a description of what the user wants to cook. This could be ingredients, flavours, utensils available, number of diners, type of meal etc.\n* negatives - a description of what the user wants to avoid in the recipe. This could be flavours to avoid, allergen considerations, conflicts with theme of meal etc.",
"inputSchema": "{\n\"type\": \"object\",\n\"properties\": {\n\t\"positive\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"a description of what the user wants to cook. This could be ingredients, flavours, utensils available, number of diners, type of meal etc.\"\n\t},\n \"negative\": {\n \"type\": \"string\",\n \"description\": \"a description of what the user wants to avoid in the recipe. This could be flavours to avoid, allergen considerations, conflicts with theme of meal etc.\"\n }\n}\n}",
"specifyInputSchema": true
},
"typeVersion": 1.1
},
{
"id": "5e703134-4dd9-464b-9ec9-dc6103907a1e",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
2420,
940
],
"parameters": {},
"typeVersion": 1
},
{
"id": "9fb5f4fd-3b38-4a35-8986-d3955754c8d1",
"name": "Mistral Cloud Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatMistralCloud",
"position": [
2660,
540
],
"parameters": {
"model": "mistral-large-2402",
"options": {}
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 1
},
{
"id": "d38275e6-aede-4f1c-9b05-018f3cf4faab",
"name": "Get Tool Response",
"type": "n8n-nodes-base.set",
"position": [
3160,
940
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "10b55200-4610-4e9b-8be7-d487c6b56a78",
"name": "response",
"type": "string",
"value": "={{ JSON.stringify($json.result) }}"
}
]
}
},
"typeVersion": 3.3
},
{
"id": "dc3ceb2f-3c64-4b42-aeca-ddcdb84abf12",
"name": "Wait for Rate Limits",
"type": "n8n-nodes-base.wait",
"position": [
2420,
1080
],
"webhookId": "e86d8ae4-3b0d-4c40-9d12-a11d6501a043",
"parameters": {
"amount": 1.1
},
"typeVersion": 1.1
},
{
"id": "ec36d6f8-c3da-4732-8d56-a092a3358864",
"name": "Get Mistral Embeddings",
"type": "n8n-nodes-base.httpRequest",
"position": [
2620,
940
],
"parameters": {
"url": "https://api.mistral.ai/v1/embeddings",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "model",
"value": "mistral-embed"
},
{
"name": "encoding_format",
"value": "float"
},
{
"name": "input",
"value": "={{ [$json.query.positive, $json.query.negative].compact() }}"
}
]
},
"nodeCredentialType": "mistralCloudApi"
},
"credentials": {
"mistralCloudApi": {
"id": "EIl2QxhXAS9Hkg37",
"name": "Mistral Cloud account"
}
},
"typeVersion": 4.2
},
{
"id": "aebcb860-d25c-4833-9e9d-0297101259c7",
"name": "Use Qdrant Recommend API",
"type": "n8n-nodes-base.httpRequest",
"position": [
2800,
940
],
"parameters": {
"url": "=http://qdrant:6333/collections/hello_fresh/points/recommend/groups",
"method": "POST",
"options": {},
"sendBody": true,
"authentication": "predefinedCredentialType",
"bodyParameters": {
"parameters": [
{
"name": "strategy",
"value": "average_vector"
},
{
"name": "limit",
"value": "={{ 3 }}"
},
{
"name": "positive",
"value": "={{ [$json.data[0].embedding] }}"
},
{
"name": "negative",
"value": "={{ [$json.data[1].embedding] }}"
},
{
"name": "filter",
"value": "={{ { \"must\": {\"key\": \"metadata.week\", \"match\": { \"value\": $('Execute Workflow Trigger').item.json.week } } } }}"
},
{
"name": "with_payload",
"value": "={{ true }}"
},
{
"name": "group_by",
"value": "metadata.recipe_id"
},
{
"name": "group_size",
"value": "={{ 3 }}"
}
]
},
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "2474c97d-0d85-4acc-a95e-2eb6494786dc",
"name": "Get Recipes From DB",
"type": "n8n-nodes-base.code",
"position": [
2980,
940
],
"parameters": {
"language": "python",
"pythonCode": "import sqlite3\ncon = sqlite3.connect(\"hello_fresh_1.db\")\n\nrecipe_ids = list(set([group.id for group in _input.all()[0].json.result.groups if group.hits[0].score > 0.5]))\nplaceholders = ','.join(['?' for i in range(0,len(recipe_ids))])\n\ncur = con.cursor()\nres = cur.execute(f'SELECT * FROM recipes WHERE id IN ({placeholders})', recipe_ids)\nrows = res.fetchall()\n\ncon.close()\n\nreturn [{ \"result\": [row[2] for row in rows] }]"
},
"typeVersion": 2
},
{
"id": "54229c2a-6e26-4350-8a94-57f415ef2340",
"name": "Save Recipes to DB",
"type": "n8n-nodes-base.code",
"position": [
1960,
940
],
"parameters": {
"language": "python",
"pythonCode": "import sqlite3\ncon = sqlite3.connect(\"hello_fresh_1.db\")\n\ncur = con.cursor()\ncur.execute(\"CREATE TABLE IF NOT EXISTS recipes (id TEXT PRIMARY KEY, name TEXT, data TEXT, cuisine TEXT, category TEXT, tag TEXT, week TEXT);\")\n\nfor item in _input.all():\n cur.execute('INSERT OR REPLACE INTO recipes VALUES(?,?,?,?,?,?,?)', (\n item.json.id,\n item.json.name,\n item.json.data,\n ','.join(item.json.cuisine),\n item.json.category,\n ','.join(item.json.tag),\n item.json.week\n ))\n\ncon.commit()\ncon.close()\n\nreturn [{ \"affected_rows\": len(_input.all()) }]"
},
"typeVersion": 2
},
{
"id": "725c1f56-5373-4891-92b9-3f32dd28892b",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
901.1666225087287,
180.99920515712074
],
"parameters": {
"color": 7,
"width": 484.12381677448207,
"height": 674.1153489831718,
"content": "## Step 1. Fetch Available Courses For the Current Week\n\nTo populate our vectorstore, we'll scrape the weekly menu off the HelloFresh Website. The pages are quite large so may take a while so please be patient."
},
"typeVersion": 1
},
{
"id": "f4e882b8-3762-4e6b-9e95-b0d708d0c284",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1420,
300
],
"parameters": {
"color": 7,
"width": 409.1756468632768,
"height": 398.81415970574335,
"content": "## Step 2. Create Recipe Documents For VectorStore\n\nTo populate our vectorstore, we'll scrape the weekly menu off the HelloFresh Website. The pages are quite large so may take a while so please be patient."
},
"typeVersion": 1
},
{
"id": "fc3c2221-b67c-451c-9096-d6acd2a297fa",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
19.326425127730317
],
"parameters": {
"color": 7,
"width": 486.02284096214964,
"height": 690.7816167755491,
"content": "## Step 3. Vectorise Recipes For Recommendation Engine\n[Read more about Qdrant node](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreqdrant)\n\nWe'll store our documents in our Qdrant vectorstore by converting to vectors using Mistral Embed. Our goal is to a build a recommendation engine for meals of the week which Qdrant is a perfect solution."
},
"typeVersion": 1
},
{
"id": "43296173-b929-46cc-b6ea-58007837b8df",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1740,
740
],
"parameters": {
"color": 7,
"width": 547.0098868353456,
"height": 347.6002738958705,
"content": "## Step 4. Save Original Document to Database\n[Read more about Code Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.code)\n\nFinally, let's have the original document stored in a more traditional datastore. USually our vectorsearch will return partial docs and those are enough for many use-cases, however in this instance we'll pull the full docs for the Agent get the info required to make the recommendation. "
},
"typeVersion": 1
},
{
"id": "6e2e58d2-e0ad-4503-8ed6-891124c8035b",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2380,
160
],
"parameters": {
"color": 7,
"width": 673.6008766895472,
"height": 552.9202706743265,
"content": "## 5. Chat with Our HelloFresh Recommendation AI Agent\n[Read more about AI Agents](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent)\n\nThis agent is designed to recommend HelloFresh recipes based on your current taste preferences. Need something hot and spicy, warm and comforting or fast and chilled? This agent will capture what you would like and not like and queries our Recipe Recommendation engine powered by Qdrant Vectorstore."
},
"typeVersion": 1
},
{
"id": "ba692c21-38bc-48a1-8b40-bad298be8b9e",
"name": "AI Agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
2660,
360
],
"parameters": {
"options": {
"systemMessage": "=You are a recipe bot for the company, \"Hello fresh\". You will help the user choose which Hello Fresh recipe to choose from this week's menu. The current week is {{ $now.year }}-W{{ $now.weekNumber }}.\nDo not recommend any recipes other from the current week's menu. If there are no recipes to recommend, please ask the user to visit the website instead https://hellofresh.com."
}
},
"typeVersion": 1.6
},
{
"id": "d7ca0f97-72dc-4f4c-8b46-3ff57b9068a4",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2320,
740
],
"parameters": {
"color": 7,
"width": 987.4785537889618,
"height": 531.9173034334732,
"content": "## 5. Using Qdrant's Recommend API & Grouping Functionality\n[Read more about Qdrant's Recommend API](https://qdrant.tech/documentation/concepts/explore/?q=recommend)\n\nUnlike basic similarity search, Qdrant's Recommend API takes a positive query to match against (eg. Roast Dinner) and a negative query to avoid (eg. Roast Chicken). This feature significantly improves results for a recommendation engine. Additionally, by utilising Qdrant's Grouping feature, we're able to group similar matches from the same recipe - meaning we can ensure unique recipes everytime."
},
"typeVersion": 1
},
{
"id": "96a294e2-1437-4ded-9973-0999b444c999",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-40
],
"parameters": {
"width": 432.916474478624,
"height": 542.9295980774649,
"content": "## Try it out!\n### This workflow does the following:\n* Fetches and stores this week's HelloFresh's menu\n* Builds the foundation of a recommendation engine by storing the recipes in a Qdrant Vectorstore and SQLite database.\n* Builds an AI Agent that allows for a chat interface to query for a the week's recipe recommendations.\n* AI agent uses the Qdrant Recommend API, providing what the user likes/dislikes as the query.\n* Qdrant returns the results which enable the AI Agent to make the recommendation to the user.\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "72c98600-f21a-42d4-97be-836b8ef6dc77",
"name": "Qdrant Vector Store",
"type": "@n8n/n8n-nodes-langchain.vectorStoreQdrant",
"position": [
1960,
240
],
"parameters": {
"mode": "insert",
"options": {},
"qdrantCollection": {
"__rl": true,
"mode": "list",
"value": "hello_fresh",
"cachedResultName": "hello_fresh"
}
},
"credentials": {
"qdrantApi": {
"id": "NyinAS3Pgfik66w5",
"name": "QdrantApi account"
}
},
"typeVersion": 1
},
{
"id": "b7c4b597-ac2b-41d7-8f0f-1cbba25085de",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
-195.8987124522777
],
"parameters": {
"width": 382.47301504497716,
"height": 195.8987124522777,
"content": "### 🚨Ensure Qdrant collection exists!\nYou'll need to run the following command in Qdrant:\n```\nPUT collections/hello_fresh\n{\n \"vectors\": {\n \"distance\": \"Cosine\",\n \"size\": 1024\n }\n}\n```"
},
"typeVersion": 1
},
{
"id": "39191834-ecc2-46f0-a31a-0a7e9c47ac5d",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
2740,
920
],
"parameters": {
"width": 213.30551928619226,
"height": 332.38559808882246,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n### 🚨Configure Your Qdrant Connection\n* Be sure to enter your endpoint address"
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Get Recipe": {
"main": [
[
{
"node": "Extract Recipe Details",
"type": "main",
"index": 0
}
]
]
},
"Chat Trigger": {
"main": [
[
{
"node": "AI Agent",
"type": "main",
"index": 0
}
]
]
},
"Prepare Documents": {
"main": [
[
{
"node": "Qdrant Vector Store",
"type": "main",
"index": 0
},
{
"node": "Save Recipes to DB",
"type": "main",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_document",
"index": 0
}
]
]
},
"Extract Server Data": {
"main": [
[
{
"node": "Extract Available Courses",
"type": "main",
"index": 0
}
]
]
},
"Get Course Metadata": {
"main": [
[
{
"node": "Merge Course & Recipe",
"type": "main",
"index": 0
}
]
]
},
"Get Recipes From DB": {
"main": [
[
{
"node": "Get Tool Response",
"type": "main",
"index": 0
}
]
]
},
"Get This Week's Menu": {
"main": [
[
{
"node": "Extract Server Data",
"type": "main",
"index": 0
}
]
]
},
"Qdrant Recommend API": {
"ai_tool": [
[
{
"node": "AI Agent",
"type": "ai_tool",
"index": 0
}
]
]
},
"Wait for Rate Limits": {
"main": [
[
{
"node": "Get Mistral Embeddings",
"type": "main",
"index": 0
}
]
]
},
"Merge Course & Recipe": {
"main": [
[
{
"node": "Prepare Documents",
"type": "main",
"index": 0
}
]
]
},
"Extract Recipe Details": {
"main": [
[
{
"node": "Merge Course & Recipe",
"type": "main",
"index": 1
}
]
]
},
"Get Mistral Embeddings": {
"main": [
[
{
"node": "Use Qdrant Recommend API",
"type": "main",
"index": 0
}
]
]
},
"Embeddings Mistral Cloud": {
"ai_embedding": [
[
{
"node": "Qdrant Vector Store",
"type": "ai_embedding",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Wait for Rate Limits",
"type": "main",
"index": 0
}
]
]
},
"Mistral Cloud Chat Model": {
"ai_languageModel": [
[
{
"node": "AI Agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Use Qdrant Recommend API": {
"main": [
[
{
"node": "Get Recipes From DB",
"type": "main",
"index": 0
}
]
]
},
"Extract Available Courses": {
"main": [
[
{
"node": "Get Course Metadata",
"type": "main",
"index": 0
},
{
"node": "Get Recipe",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Test workflow\"": {
"main": [
[
{
"node": "Get This Week's Menu",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}sRecipe Recommendations with Qdrant and Mistral

394
AI_Research_RAG_and_Data_Analysis/Reconcile Rent Payments with Local Excel Spreadsheet and OpenAI.json Normal file
View File

@@ -0,0 +1,394 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "bebbf9cf-8103-4694-a3be-ae3ee1e9ebaf",
"name": "Watch For Bank Statements",
"type": "n8n-nodes-base.localFileTrigger",
"position": [
780,
400
],
"parameters": {
"path": "/home/node/host_mount/reconciliation_project",
"events": [
"add"
],
"options": {
"ignored": "!**/*.csv"
},
"triggerOn": "folder"
},
"typeVersion": 1
},
{
"id": "eca26bed-ba44-4507-97d4-9154e26908a5",
"name": "Get Tenant Details",
"type": "@n8n/n8n-nodes-langchain.toolCode",
"position": [
1660,
540
],
"parameters": {
"name": "get_tenant_details",
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').item.json;\nconst sheetName = 'tenants';\n\nconst wb = xlsx.readFile(spreadsheet_location, { sheets: [sheetName] });\nconst rows = xlsx.utils.sheet_to_json(wb.Sheets[sheetName], { raw: false });\n\nconst queryToList = [].concat(typeof query === 'string' ? query.split(',') : query);\n\nconst result = queryToList.map(q => (\n rows.find(row =>\n row['Tenant Name'].toLowerCase() === q.toLowerCase()\n || row['Tenant ID'].toLowerCase() === q.toString().toLowerCase()\n )\n));\n\nreturn result ? JSON.stringify(result) : `No results were found for ${query}`;",
"description": "Call this tool to get a tenant's details which includes their tenancy terms, rent amount and any notes attached to their account. Pass in one or an array of either the tenant ID or the name of the tenant."
},
"typeVersion": 1.1
},
{
"id": "76b68c2f-8d33-4f61-a442-732e784b733a",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
1920,
540
],
"parameters": {
"jsonSchemaExample": "[{\n \"tenant_id\": \"\",\n \"tenant_name\": \"\",\n \"property_id\": \"\",\n \"property_postcode\": \"\",\n \"action_required\": \"\",\n \"details\": \"\",\n \"date\": \"\"\n}]"
},
"typeVersion": 1.2
},
{
"id": "be01720f-4617-4a2b-aaed-2474f9f0e25b",
"name": "Get Bank Statement File",
"type": "n8n-nodes-base.readWriteFile",
"position": [
1100,
400
],
"parameters": {
"options": {},
"fileSelector": "={{ $('Watch For Bank Statements').item.json.path }}"
},
"typeVersion": 1
},
{
"id": "2aba5f6a-56b0-411f-9124-33025d90e325",
"name": "Get CSV Data",
"type": "n8n-nodes-base.extractFromFile",
"position": [
1260,
400
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "a60d5851-f938-4696-855b-1f0845ffbc6c",
"name": "Alert Actions To List",
"type": "n8n-nodes-base.splitOut",
"position": [
2260,
400
],
"parameters": {
"options": {},
"fieldToSplitOut": "output"
},
"typeVersion": 1
},
{
"id": "f804d9fb-f679-4e95-b70f-722e7c222c40",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
690.6721905682555,
177.80249392766257
],
"parameters": {
"color": 7,
"width": 748.2548372021405,
"height": 457.6238063670572,
"content": "## Step 1. Wait For Incoming Bank Statements\n[Read more about the local file triggers](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.localfiletrigger)\n\nFor this demo, we'll show that n8n is more than capable working with the local filesystem. This gives great benefits in terms of privacy and data security.\n\nFor our datastore, we're using a locally hosted XLSX Excel file which we'll query and update throughout this workflow."
},
"typeVersion": 1
},
{
"id": "01e9c335-320c-4fff-9ade-ad1cf808db00",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1460,
80
],
"parameters": {
"color": 7,
"width": 634.3165117416636,
"height": 675.2455596085985,
"content": "## Step 2. Delegate to AI Agent to Quickly Identify Issues with Rental Payments\n[Read more about AI Agents](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent/)\n\nAn AI agent can not only check against agreed amounts and compare due dates but also consider contract exceptions and tenant notes before deciding to take action. In a scenario of 10+ of tenants, this can save a lot of admin time.\n\nFor this demo, we're using a remote LLM Model but this can easily be swapped out for other self-hosted LLMS models that support function calling."
},
"typeVersion": 1
},
{
"id": "2456b1e5-ceec-45c3-91a7-52e21125e6e5",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
143.8836673253448
],
"parameters": {
"color": 7,
"width": 618.3293247808133,
"height": 473.7439917476675,
"content": "## Step 3. Generate a Report to Action any Issues\n[Read more about using the Code Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.code)\n\nAfter the AI Agent has helped identify issues to action, we can generate a report and update a locally hosted xlsx file. This again helps keep workflows private to nothing senstive goes over the wire.\n\nThough n8n lacks a builtin node for editing local xlsx file, we can tap into the sheetJS library available to the \"Code\" node."
},
"typeVersion": 1
},
{
"id": "7b32e8f9-b543-47e1-a08e-53ee47105966",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
260,
80
],
"parameters": {
"width": 399.5148533727183,
"height": 558.2628336538015,
"content": "## Try It Out!\n### This workflow ingests bank statements to analyses them against a list of tenants using an AI Agent. The agent then flags any issues such as missing payments or incorrect amounts which are exported to a XLSX spreadsheet.\n\n### Note: This workflow is intended to work with a self-hosted version of n8n and has access to the local file system.\n\n* Watches for CSV files (bank statements)\n* Imports into AI agent for analysis.\n* AI agent will query the Excel spreadsheet for tenant and property details.\n* AI agent will generate report on discrepancies or issues and write them to the Excel file.\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "ba35ed0b-7ace-4b76-b915-0dc516a07fb1",
"name": "Get Property Details",
"type": "@n8n/n8n-nodes-langchain.toolCode",
"position": [
1800,
540
],
"parameters": {
"name": "get_property_details",
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').item.json;\nconst sheetName = 'properties'\n\nconst wb = xlsx.readFile(spreadsheet_location, { sheets: [sheetName] });\nconst rows = xlsx.utils.sheet_to_json(wb.Sheets[sheetName], { raw: false });\n\nconst queryToList = [].concat(typeof query === 'string' ? query.split(',') :query);\n\nconst result = queryToList.map(q => rows.find(row => row['Property ID'] === q));\n\nreturn result ? JSON.stringify(result) : `No results were found for ${query}`;",
"description": "Call this tool to get a property details which includes the address, postcode and type of the property. Pass in one or an array of Property IDs."
},
"typeVersion": 1.1
},
{
"id": "8c85a2f5-6741-41f4-b377-c74a74b14d0f",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
940,
400
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bcd3dd04-0082-4da6-b36b-e5ad09c4de30",
"name": "spreadsheet_location",
"type": "string",
"value": "/home/node/host_mount/reconciliation_project/reconcilation-workbook.xlsx"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "bd75bad8-caa3-48f1-8892-3d1221765564",
"name": "Append To Spreadsheet",
"type": "n8n-nodes-base.code",
"position": [
2480,
400
],
"parameters": {
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').first().json;\nconst sheetName = 'alerts';\n\nconst wb = xlsx.readFile(spreadsheet_location);\nxlsx.writeFile(wb, spreadsheet_location + '.bak.xlsx'); // create backup\n\nconst worksheet = wb.Sheets[sheetName];\n\nconst inputs = $input.all();\n\nfor (input of inputs) {\n xlsx.utils.sheet_add_aoa(worksheet, [\n [\n input.json.date,\n input.json[\"property_id\"],\n input.json[\"property_postcode\"],\n input.json[\"tenant_id\"],\n input.json[\"tenant_name\"],\n input.json[\"action_required\"],\n input.json[\"details\"],\n ]\n ], { origin: -1 });\n}\n\n// update sheet ref\nconst range = xlsx.utils.decode_range(worksheet['!ref']);\nconst rowIndex = range.e.r + 1; // The next row index to append\nworksheet['!ref'] = xlsx.utils.encode_range({\n s: range.s,\n e: { r: rowIndex, c: range.e.c }\n});\n\nxlsx.writeFile(wb, spreadsheet_location, {\n cellDates: true,\n cellStyles: true,\n bookType: 'xlsx',\n});\n\nreturn {\"json\": { \"output\": `${inputs.length} rows added` }}"
},
"typeVersion": 2
},
{
"id": "c818ea7e-dc57-4680-b797-abb21cca87fb",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
1540,
540
],
"parameters": {
"model": "gpt-4o",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "b2a97514-6020-49a6-bbdb-ee1251eb6aed",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2280,
640
],
"parameters": {
"color": 3,
"width": 461.5505566920007,
"height": 106.59049079746408,
"content": "### 🚨Warning! Potentially Destructive Operations!\nWith code comes great responsibility! There is a risk you may overwrite/delete data you didn't intend. Always makes backups and test on a copy of your spreadsheets!"
},
"typeVersion": 1
},
{
"id": "f869f6eb-cf19-4b14-bf3a-4db5d636646f",
"name": "Reconcile Rental Payments",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
1640,
360
],
"parameters": {
"text": "=Bank Statement for {{ $input.first().json.date }} to {{ $input.last().json.date }}:\n|date|reference|money in|money out|\n|-|-|-|-|\n{{ $input.all().map(row => `|${row.json.date}|${row.json.reference}|${row.json.money_in || ''}|${row.json.money_out || ''}|`).join('\\n') }}",
"options": {
"systemMessage": "Your task is to help reconcile rent payments with the uploaded bank statement and alert only if there are any actions to be taken in regards to the tenants.\n* Identify and flag any tenants who have have missed their rent according to the month. Late payments which are within a few days of the due date are acceptable and should not be flagged.\n* Identify and flag if any tenants have not paid the correct ammount due, either less or more.\n* Identify and flag any tenants who are finishing their rentals within the time period of the current statement.\n* Identify and flag any remaining fees which are due and have not been paid from any tenant in the last month of their rental.\n\nIf the bank statement show incomplete months due to cut off, it is ok to assume the payment is pending and not actually missing.\n\nThe alert system requires a JSON formatted message. It is important that you format your response as follows:\n[{\n \"tenant_id\": \"\",\n \"tenant_name\": \"\",\n \"property_id\": \"\",\n \"property_postcode\": \"\",\n \"action required\": \"\",\n \"details\": \"\",\n \"date\": \"\"\n}]"
},
"promptType": "define",
"hasOutputParser": true
},
"executeOnce": true,
"typeVersion": 1.6
},
{
"id": "510dc73c-f267-41f3-a981-58f5bfc229a6",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
360,
660
],
"parameters": {
"color": 5,
"width": 302.6142384407349,
"height": 86.00673806595168,
"content": "### 💡I'm designed to work self-hosted!\nSome nodes in this workflow are only available to the self-hosted version of n8n."
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Get CSV Data": {
"main": [
[
{
"node": "Reconcile Rental Payments",
"type": "main",
"index": 0
}
]
]
},
"Set Variables": {
"main": [
[
{
"node": "Get Bank Statement File",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Get Tenant Details": {
"ai_tool": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_tool",
"index": 0
}
]
]
},
"Get Property Details": {
"ai_tool": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_tool",
"index": 0
}
]
]
},
"Alert Actions To List": {
"main": [
[
{
"node": "Append To Spreadsheet",
"type": "main",
"index": 0
}
]
]
},
"Get Bank Statement File": {
"main": [
[
{
"node": "Get CSV Data",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Reconcile Rental Payments": {
"main": [
[
{
"node": "Alert Actions To List",
"type": "main",
"index": 0
}
]
]
},
"Watch For Bank Statements": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
}
}
}

394
AI_Research_RAG_and_Data_Analysis/Reconcile Rent Payments with Local Excel Spreadsheet and OpenAI.txt
View File

@@ -1,394 +0,0 @@
{
"meta": {
"instanceId": "26ba763460b97c249b82942b23b6384876dfeb9327513332e743c5f6219c2b8e"
},
"nodes": [
{
"id": "bebbf9cf-8103-4694-a3be-ae3ee1e9ebaf",
"name": "Watch For Bank Statements",
"type": "n8n-nodes-base.localFileTrigger",
"position": [
780,
400
],
"parameters": {
"path": "/home/node/host_mount/reconciliation_project",
"events": [
"add"
],
"options": {
"ignored": "!**/*.csv"
},
"triggerOn": "folder"
},
"typeVersion": 1
},
{
"id": "eca26bed-ba44-4507-97d4-9154e26908a5",
"name": "Get Tenant Details",
"type": "@n8n/n8n-nodes-langchain.toolCode",
"position": [
1660,
540
],
"parameters": {
"name": "get_tenant_details",
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').item.json;\nconst sheetName = 'tenants';\n\nconst wb = xlsx.readFile(spreadsheet_location, { sheets: [sheetName] });\nconst rows = xlsx.utils.sheet_to_json(wb.Sheets[sheetName], { raw: false });\n\nconst queryToList = [].concat(typeof query === 'string' ? query.split(',') : query);\n\nconst result = queryToList.map(q => (\n rows.find(row =>\n row['Tenant Name'].toLowerCase() === q.toLowerCase()\n || row['Tenant ID'].toLowerCase() === q.toString().toLowerCase()\n )\n));\n\nreturn result ? JSON.stringify(result) : `No results were found for ${query}`;",
"description": "Call this tool to get a tenant's details which includes their tenancy terms, rent amount and any notes attached to their account. Pass in one or an array of either the tenant ID or the name of the tenant."
},
"typeVersion": 1.1
},
{
"id": "76b68c2f-8d33-4f61-a442-732e784b733a",
"name": "Structured Output Parser",
"type": "@n8n/n8n-nodes-langchain.outputParserStructured",
"position": [
1920,
540
],
"parameters": {
"jsonSchemaExample": "[{\n \"tenant_id\": \"\",\n \"tenant_name\": \"\",\n \"property_id\": \"\",\n \"property_postcode\": \"\",\n \"action_required\": \"\",\n \"details\": \"\",\n \"date\": \"\"\n}]"
},
"typeVersion": 1.2
},
{
"id": "be01720f-4617-4a2b-aaed-2474f9f0e25b",
"name": "Get Bank Statement File",
"type": "n8n-nodes-base.readWriteFile",
"position": [
1100,
400
],
"parameters": {
"options": {},
"fileSelector": "={{ $('Watch For Bank Statements').item.json.path }}"
},
"typeVersion": 1
},
{
"id": "2aba5f6a-56b0-411f-9124-33025d90e325",
"name": "Get CSV Data",
"type": "n8n-nodes-base.extractFromFile",
"position": [
1260,
400
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "a60d5851-f938-4696-855b-1f0845ffbc6c",
"name": "Alert Actions To List",
"type": "n8n-nodes-base.splitOut",
"position": [
2260,
400
],
"parameters": {
"options": {},
"fieldToSplitOut": "output"
},
"typeVersion": 1
},
{
"id": "f804d9fb-f679-4e95-b70f-722e7c222c40",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
690.6721905682555,
177.80249392766257
],
"parameters": {
"color": 7,
"width": 748.2548372021405,
"height": 457.6238063670572,
"content": "## Step 1. Wait For Incoming Bank Statements\n[Read more about the local file triggers](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.localfiletrigger)\n\nFor this demo, we'll show that n8n is more than capable working with the local filesystem. This gives great benefits in terms of privacy and data security.\n\nFor our datastore, we're using a locally hosted XLSX Excel file which we'll query and update throughout this workflow."
},
"typeVersion": 1
},
{
"id": "01e9c335-320c-4fff-9ade-ad1cf808db00",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
1460,
80
],
"parameters": {
"color": 7,
"width": 634.3165117416636,
"height": 675.2455596085985,
"content": "## Step 2. Delegate to AI Agent to Quickly Identify Issues with Rental Payments\n[Read more about AI Agents](https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent/)\n\nAn AI agent can not only check against agreed amounts and compare due dates but also consider contract exceptions and tenant notes before deciding to take action. In a scenario of 10+ of tenants, this can save a lot of admin time.\n\nFor this demo, we're using a remote LLM Model but this can easily be swapped out for other self-hosted LLMS models that support function calling."
},
"typeVersion": 1
},
{
"id": "2456b1e5-ceec-45c3-91a7-52e21125e6e5",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
143.8836673253448
],
"parameters": {
"color": 7,
"width": 618.3293247808133,
"height": 473.7439917476675,
"content": "## Step 3. Generate a Report to Action any Issues\n[Read more about using the Code Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.code)\n\nAfter the AI Agent has helped identify issues to action, we can generate a report and update a locally hosted xlsx file. This again helps keep workflows private to nothing senstive goes over the wire.\n\nThough n8n lacks a builtin node for editing local xlsx file, we can tap into the sheetJS library available to the \"Code\" node."
},
"typeVersion": 1
},
{
"id": "7b32e8f9-b543-47e1-a08e-53ee47105966",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
260,
80
],
"parameters": {
"width": 399.5148533727183,
"height": 558.2628336538015,
"content": "## Try It Out!\n### This workflow ingests bank statements to analyses them against a list of tenants using an AI Agent. The agent then flags any issues such as missing payments or incorrect amounts which are exported to a XLSX spreadsheet.\n\n### Note: This workflow is intended to work with a self-hosted version of n8n and has access to the local file system.\n\n* Watches for CSV files (bank statements)\n* Imports into AI agent for analysis.\n* AI agent will query the Excel spreadsheet for tenant and property details.\n* AI agent will generate report on discrepancies or issues and write them to the Excel file.\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
},
"typeVersion": 1
},
{
"id": "ba35ed0b-7ace-4b76-b915-0dc516a07fb1",
"name": "Get Property Details",
"type": "@n8n/n8n-nodes-langchain.toolCode",
"position": [
1800,
540
],
"parameters": {
"name": "get_property_details",
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').item.json;\nconst sheetName = 'properties'\n\nconst wb = xlsx.readFile(spreadsheet_location, { sheets: [sheetName] });\nconst rows = xlsx.utils.sheet_to_json(wb.Sheets[sheetName], { raw: false });\n\nconst queryToList = [].concat(typeof query === 'string' ? query.split(',') :query);\n\nconst result = queryToList.map(q => rows.find(row => row['Property ID'] === q));\n\nreturn result ? JSON.stringify(result) : `No results were found for ${query}`;",
"description": "Call this tool to get a property details which includes the address, postcode and type of the property. Pass in one or an array of Property IDs."
},
"typeVersion": 1.1
},
{
"id": "8c85a2f5-6741-41f4-b377-c74a74b14d0f",
"name": "Set Variables",
"type": "n8n-nodes-base.set",
"position": [
940,
400
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bcd3dd04-0082-4da6-b36b-e5ad09c4de30",
"name": "spreadsheet_location",
"type": "string",
"value": "/home/node/host_mount/reconciliation_project/reconcilation-workbook.xlsx"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "bd75bad8-caa3-48f1-8892-3d1221765564",
"name": "Append To Spreadsheet",
"type": "n8n-nodes-base.code",
"position": [
2480,
400
],
"parameters": {
"jsCode": "const xlsx = require('xlsx');\n\nconst { spreadsheet_location } = $('Set Variables').first().json;\nconst sheetName = 'alerts';\n\nconst wb = xlsx.readFile(spreadsheet_location);\nxlsx.writeFile(wb, spreadsheet_location + '.bak.xlsx'); // create backup\n\nconst worksheet = wb.Sheets[sheetName];\n\nconst inputs = $input.all();\n\nfor (input of inputs) {\n xlsx.utils.sheet_add_aoa(worksheet, [\n [\n input.json.date,\n input.json[\"property_id\"],\n input.json[\"property_postcode\"],\n input.json[\"tenant_id\"],\n input.json[\"tenant_name\"],\n input.json[\"action_required\"],\n input.json[\"details\"],\n ]\n ], { origin: -1 });\n}\n\n// update sheet ref\nconst range = xlsx.utils.decode_range(worksheet['!ref']);\nconst rowIndex = range.e.r + 1; // The next row index to append\nworksheet['!ref'] = xlsx.utils.encode_range({\n s: range.s,\n e: { r: rowIndex, c: range.e.c }\n});\n\nxlsx.writeFile(wb, spreadsheet_location, {\n cellDates: true,\n cellStyles: true,\n bookType: 'xlsx',\n});\n\nreturn {\"json\": { \"output\": `${inputs.length} rows added` }}"
},
"typeVersion": 2
},
{
"id": "c818ea7e-dc57-4680-b797-abb21cca87fb",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
1540,
540
],
"parameters": {
"model": "gpt-4o",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "b2a97514-6020-49a6-bbdb-ee1251eb6aed",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2280,
640
],
"parameters": {
"color": 3,
"width": 461.5505566920007,
"height": 106.59049079746408,
"content": "### 🚨Warning! Potentially Destructive Operations!\nWith code comes great responsibility! There is a risk you may overwrite/delete data you didn't intend. Always makes backups and test on a copy of your spreadsheets!"
},
"typeVersion": 1
},
{
"id": "f869f6eb-cf19-4b14-bf3a-4db5d636646f",
"name": "Reconcile Rental Payments",
"type": "@n8n/n8n-nodes-langchain.agent",
"position": [
1640,
360
],
"parameters": {
"text": "=Bank Statement for {{ $input.first().json.date }} to {{ $input.last().json.date }}:\n|date|reference|money in|money out|\n|-|-|-|-|\n{{ $input.all().map(row => `|${row.json.date}|${row.json.reference}|${row.json.money_in || ''}|${row.json.money_out || ''}|`).join('\\n') }}",
"options": {
"systemMessage": "Your task is to help reconcile rent payments with the uploaded bank statement and alert only if there are any actions to be taken in regards to the tenants.\n* Identify and flag any tenants who have have missed their rent according to the month. Late payments which are within a few days of the due date are acceptable and should not be flagged.\n* Identify and flag if any tenants have not paid the correct ammount due, either less or more.\n* Identify and flag any tenants who are finishing their rentals within the time period of the current statement.\n* Identify and flag any remaining fees which are due and have not been paid from any tenant in the last month of their rental.\n\nIf the bank statement show incomplete months due to cut off, it is ok to assume the payment is pending and not actually missing.\n\nThe alert system requires a JSON formatted message. It is important that you format your response as follows:\n[{\n \"tenant_id\": \"\",\n \"tenant_name\": \"\",\n \"property_id\": \"\",\n \"property_postcode\": \"\",\n \"action required\": \"\",\n \"details\": \"\",\n \"date\": \"\"\n}]"
},
"promptType": "define",
"hasOutputParser": true
},
"executeOnce": true,
"typeVersion": 1.6
},
{
"id": "510dc73c-f267-41f3-a981-58f5bfc229a6",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
360,
660
],
"parameters": {
"color": 5,
"width": 302.6142384407349,
"height": 86.00673806595168,
"content": "### 💡I'm designed to work self-hosted!\nSome nodes in this workflow are only available to the self-hosted version of n8n."
},
"typeVersion": 1
}
],
"pinData": {},
"connections": {
"Get CSV Data": {
"main": [
[
{
"node": "Reconcile Rental Payments",
"type": "main",
"index": 0
}
]
]
},
"Set Variables": {
"main": [
[
{
"node": "Get Bank Statement File",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Get Tenant Details": {
"ai_tool": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_tool",
"index": 0
}
]
]
},
"Get Property Details": {
"ai_tool": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_tool",
"index": 0
}
]
]
},
"Alert Actions To List": {
"main": [
[
{
"node": "Append To Spreadsheet",
"type": "main",
"index": 0
}
]
]
},
"Get Bank Statement File": {
"main": [
[
{
"node": "Get CSV Data",
"type": "main",
"index": 0
}
]
]
},
"Structured Output Parser": {
"ai_outputParser": [
[
{
"node": "Reconcile Rental Payments",
"type": "ai_outputParser",
"index": 0
}
]
]
},
"Reconcile Rental Payments": {
"main": [
[
{
"node": "Alert Actions To List",
"type": "main",
"index": 0
}
]
]
},
"Watch For Bank Statements": {
"main": [
[
{
"node": "Set Variables",
"type": "main",
"index": 0
}
]
]
}
}
}

920
AI_Research_RAG_and_Data_Analysis/Scrape Trustpilot Reviews with DeepSeek, Analyze Sentiment with OpenAI.json Normal file
View File

@@ -0,0 +1,920 @@
{
"id": "w434EiZ2z7klQAyp",
"meta": {
"instanceId": "a4bfc93e975ca233ac45ed7c9227d84cf5a2329310525917adaf3312e10d5462",
"templateCredsSetupCompleted": true
},
"name": "Scrape Trustpilot Reviews with DeepSeek, Analyze Sentiment with OpenAI",
"tags": [
{
"id": "2VG6RbmUdJ2VZbrj",
"name": "Google Drive",
"createdAt": "2024-12-04T16:50:56.177Z",
"updatedAt": "2024-12-04T16:50:56.177Z"
},
{
"id": "paTcf5QZDJsC2vKY",
"name": "OpenAI",
"createdAt": "2024-12-04T16:52:10.768Z",
"updatedAt": "2024-12-04T16:52:10.768Z"
}
],
"nodes": [
{
"id": "095a8e10-1630-4a1a-b6c9-7950ae1ed803",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
320,
-380
],
"parameters": {
"options": {},
"fieldToSplitOut": "recensioni"
},
"typeVersion": 1
},
{
"id": "6ff4dd9d-eedd-4d84-b13a-b3c0db717409",
"name": "Information Extractor",
"type": "@n8n/n8n-nodes-langchain.informationExtractor",
"position": [
-440,
140
],
"parameters": {
"text": "=You need to extract the review from the following HTML: {{ $json.recensione }}",
"options": {
"systemPromptTemplate": "You are a review expert. You need to extract only the required information and report it without changing anything.\nAll the required information is in the text."
},
"attributes": {
"attributes": [
{
"name": "autore",
"required": true,
"description": "Extract the name of the review author"
},
{
"name": "valutazione",
"type": "number",
"required": true,
"description": "Extract the rating given to the review (from 1 to 5)"
},
{
"name": "data",
"required": true,
"description": "Extract review date in YYYY-MM-DD format"
},
{
"name": "titolo",
"required": true,
"description": "Extract the review title"
},
{
"name": "testo",
"required": true,
"description": "Extract the review text"
},
{
"name": "n_recensioni",
"type": "number",
"required": true,
"description": "Extract the total number of reviews made by the user"
},
{
"name": "nazione",
"required": true,
"description": "Extract the country of the user who wrote the review. Must be two characters"
}
]
}
},
"typeVersion": 1
},
{
"id": "0036f3b1-4832-4a35-8694-0893475a4119",
"name": "If",
"type": "n8n-nodes-base.if",
"position": [
60,
-100
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "loose"
},
"combinator": "and",
"conditions": [
{
"id": "ab666549-4eec-40e2-a702-0575c094a2d4",
"operator": {
"type": "string",
"operation": "empty",
"singleValue": true
},
"leftValue": "={{ $json.Valutazione }}",
"rightValue": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}"
}
]
},
"looseTypeValidation": true
},
"executeOnce": false,
"typeVersion": 2.2
},
{
"id": "5423b55d-eb6c-41c6-9b26-410e3c92b85d",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-700,
-380
],
"parameters": {},
"typeVersion": 1
},
{
"id": "506cdaa1-e0ba-4f29-b137-69d321b13c94",
"name": "Limit1",
"type": "n8n-nodes-base.limit",
"position": [
540,
-380
],
"parameters": {
"maxItems": 3
},
"typeVersion": 1
},
{
"id": "40f1e30d-8aed-4995-b4e4-2239248bd6e7",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-460,
-480
],
"parameters": {
"width": 212.25249169435213,
"height": 245.55481727574733,
"content": "Change to the name of the company registered on Trustpilot and the maximum number of pages to scrape"
},
"typeVersion": 1
},
{
"id": "e6d2fec1-7255-4270-86b4-6d6f39f44ccb",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-460,
80
],
"parameters": {
"width": 381,
"height": 177,
"content": "Extract all information with DeepSeek (remember to change base_url with https://api.deepseek.com/v1)"
},
"typeVersion": 1
},
{
"id": "af5e962c-4faf-41cc-a8b8-2fbb145b7af6",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-160
],
"parameters": {
"width": 501.28903654485043,
"height": 195.84053156146172,
"content": "Check if the review has already been saved to Google Drive"
},
"typeVersion": 1
},
{
"id": "400dff0c-8b2e-4fe2-933e-1f4d14624ca1",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
80
],
"parameters": {
"width": 301.27574750830576,
"height": 177.34219269102988,
"content": "Analyze review sentiment"
},
"typeVersion": 1
},
{
"id": "52757ade-4206-40f9-bf4f-c3aefb004d2e",
"name": "Set Parameters",
"type": "n8n-nodes-base.set",
"position": [
-440,
-380
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "556e201d-242a-4c0e-bc13-787c2b60f800",
"name": "company_id",
"type": "string",
"value": "COMPANY"
},
{
"id": "a1f239df-df08-41d8-8b78-d6502266a581",
"name": "max_page",
"type": "number",
"value": 2
}
]
}
},
"typeVersion": 3.4
},
{
"id": "cd7e9d36-7ecd-4d9c-b552-8f46b0cfcc03",
"name": "Get reviews",
"type": "n8n-nodes-base.httpRequest",
"position": [
-200,
-380
],
"parameters": {
"url": "=https://it.trustpilot.com/review/{{ $json.company_id }}",
"options": {
"pagination": {
"pagination": {
"parameters": {
"parameters": [
{
"name": "page",
"value": "={{ $pageCount + 1 }}"
}
]
},
"maxRequests": "={{ $json.max_page }}",
"requestInterval": 5000,
"limitPagesFetched": true
}
}
},
"sendQuery": true,
"queryParameters": {
"parameters": [
{
"name": "sort",
"value": "recency"
}
]
}
},
"typeVersion": 4.2
},
{
"id": "476ff7b6-ab30-4674-a7fe-b032128ee51a",
"name": "Extract",
"type": "n8n-nodes-base.html",
"position": [
60,
-380
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "recensioni",
"attribute": "href",
"cssSelector": "article section a",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "a2a35455-7d3e-4c4c-aa66-6cbbd48d867a",
"name": "Get rows",
"type": "n8n-nodes-base.googleSheets",
"position": [
-200,
-100
],
"parameters": {
"options": {},
"filtersUI": {
"values": [
{
"lookupValue": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}",
"lookupColumn": "Id"
}
]
},
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Review"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"typeVersion": 4.5
},
{
"id": "2d507fe6-a4fc-42ff-97ff-dfd552c651ab",
"name": "Get Google Sheets",
"type": "n8n-nodes-base.googleSheets",
"position": [
-440,
-100
],
"parameters": {
"columns": {
"value": {
"Id": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}"
},
"schema": [
{
"id": "Id",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Id",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Data",
"type": "string",
"display": true,
"required": false,
"displayName": "Data",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Nome",
"type": "string",
"display": true,
"required": false,
"displayName": "Nome",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Titolo",
"type": "string",
"display": true,
"required": false,
"displayName": "Titolo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Testo",
"type": "string",
"display": true,
"required": false,
"displayName": "Testo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Località",
"type": "string",
"display": true,
"required": false,
"displayName": "Località",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "N. Recensioni",
"type": "string",
"display": true,
"required": false,
"displayName": "N. Recensioni",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "URL",
"type": "string",
"display": true,
"required": false,
"displayName": "URL",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Valutazione",
"type": "string",
"display": true,
"required": false,
"displayName": "Valutazione",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Sentiment",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Sentiment",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"Id"
],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "appendOrUpdate",
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Reviews"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"executeOnce": false,
"typeVersion": 4.5
},
{
"id": "0a1fab6e-96b7-403b-884e-f67be6e23fa5",
"name": "Get Single review",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
-120
],
"parameters": {
"url": "=https://it.trustpilot.com{{ $('Split Out').item.json.recensioni }}",
"options": {}
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "7d322d76-1032-405a-9d46-2958761a184d",
"name": "Extract review",
"type": "n8n-nodes-base.html",
"position": [
540,
-120
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "recensione",
"cssSelector": "article",
"returnArray": true
}
]
}
},
"typeVersion": 1.2
},
{
"id": "952484e5-8e87-4eb3-99a6-5bf26c701ba8",
"name": "Update sheet",
"type": "n8n-nodes-base.googleSheets",
"position": [
520,
120
],
"parameters": {
"columns": {
"value": {
"Id": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}",
"URL": "=https://it.trustpilot.com{{ $('Split Out').item.json.recensioni }}",
"Data": "={{ $('Information Extractor').item.json.output.data }}",
"Nome": "={{ $json.output.autore }}",
"Testo": "={{ $('Information Extractor').item.json.output.testo }}",
"Titolo": "={{ $('Information Extractor').item.json.output.titolo }}",
"Località": "={{ $('Information Extractor').item.json.output.nazione }}",
"Sentiment": "={{ $json.sentimentAnalysis.category }}",
"Valutazione": "={{ $('Information Extractor').item.json.output.valutazione }}",
"N. Recensioni": "={{ $('Information Extractor').item.json.output.n_recensioni }}"
},
"schema": [
{
"id": "Id",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Id",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Data",
"type": "string",
"display": true,
"required": false,
"displayName": "Data",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Nome",
"type": "string",
"display": true,
"required": false,
"displayName": "Nome",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Titolo",
"type": "string",
"display": true,
"required": false,
"displayName": "Titolo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Testo",
"type": "string",
"display": true,
"required": false,
"displayName": "Testo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Località",
"type": "string",
"display": true,
"required": false,
"displayName": "Località",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "N. Recensioni",
"type": "string",
"display": true,
"required": false,
"displayName": "N. Recensioni",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "URL",
"type": "string",
"display": true,
"required": false,
"displayName": "URL",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Valutazione",
"type": "string",
"display": true,
"required": false,
"displayName": "Valutazione",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Sentiment",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Sentiment",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"Id"
],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "appendOrUpdate",
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Reviews"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"typeVersion": 4.5
},
{
"id": "eb853885-816d-4df7-b5ac-900fa89d3df9",
"name": "Sentiment Analysis",
"type": "@n8n/n8n-nodes-langchain.sentimentAnalysis",
"position": [
60,
140
],
"parameters": {
"options": {
"categories": "Positive, Neutral, Negative",
"systemPromptTemplate": "You are highly intelligent and accurate sentiment analyzer. Analyze the sentiment of the provided text. Categorize it into one of the following: {categories}. Use the provided formatting instructions. Only output the JSON."
},
"inputText": "={{ $json.output.testo }}"
},
"typeVersion": 1
},
{
"id": "79f1b9ea-6297-4735-9c0f-9f28dd65efa0",
"name": "DeepSeek Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
-460,
320
],
"parameters": {
"model": "deepseek-reasoner",
"options": {
"baseURL": "https://api.deepseek.com/v1"
}
},
"credentials": {
"openAiApi": {
"id": "97Cz4cqyiy1RdcQL",
"name": "DeepSeek"
}
},
"typeVersion": 1
},
{
"id": "159cc88e-1dd3-4bba-a3c8-59a9aad14c88",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
40,
320
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "CDX6QM4gLYanh0P4",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "43c8ee74-159c-4217-9cb4-554c63a3b183",
"connections": {
"If": {
"main": [
[
{
"node": "Get Single review",
"type": "main",
"index": 0
}
]
]
},
"Limit1": {
"main": [
[
{
"node": "Get Google Sheets",
"type": "main",
"index": 0
}
]
]
},
"Extract": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Get rows": {
"main": [
[
{
"node": "If",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Limit1",
"type": "main",
"index": 0
}
]
]
},
"Get reviews": {
"main": [
[
{
"node": "Extract",
"type": "main",
"index": 0
}
]
]
},
"Extract review": {
"main": [
[
{
"node": "Information Extractor",
"type": "main",
"index": 0
}
]
]
},
"Set Parameters": {
"main": [
[
{
"node": "Get reviews",
"type": "main",
"index": 0
}
]
]
},
"Get Google Sheets": {
"main": [
[
{
"node": "Get rows",
"type": "main",
"index": 0
}
]
]
},
"Get Single review": {
"main": [
[
{
"node": "Extract review",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Sentiment Analysis",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Sentiment Analysis": {
"main": [
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
],
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
],
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
]
]
},
"DeepSeek Chat Model": {
"ai_languageModel": [
[
{
"node": "Information Extractor",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Information Extractor": {
"main": [
[
{
"node": "Sentiment Analysis",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Parameters",
"type": "main",
"index": 0
}
]
]
}
}
}

920
AI_Research_RAG_and_Data_Analysis/Scrape Trustpilot Reviews with DeepSeek, Analyze Sentiment with OpenAI.txt
View File

@@ -1,920 +0,0 @@
{
"id": "w434EiZ2z7klQAyp",
"meta": {
"instanceId": "a4bfc93e975ca233ac45ed7c9227d84cf5a2329310525917adaf3312e10d5462",
"templateCredsSetupCompleted": true
},
"name": "Scrape Trustpilot Reviews with DeepSeek, Analyze Sentiment with OpenAI",
"tags": [
{
"id": "2VG6RbmUdJ2VZbrj",
"name": "Google Drive",
"createdAt": "2024-12-04T16:50:56.177Z",
"updatedAt": "2024-12-04T16:50:56.177Z"
},
{
"id": "paTcf5QZDJsC2vKY",
"name": "OpenAI",
"createdAt": "2024-12-04T16:52:10.768Z",
"updatedAt": "2024-12-04T16:52:10.768Z"
}
],
"nodes": [
{
"id": "095a8e10-1630-4a1a-b6c9-7950ae1ed803",
"name": "Split Out",
"type": "n8n-nodes-base.splitOut",
"position": [
320,
-380
],
"parameters": {
"options": {},
"fieldToSplitOut": "recensioni"
},
"typeVersion": 1
},
{
"id": "6ff4dd9d-eedd-4d84-b13a-b3c0db717409",
"name": "Information Extractor",
"type": "@n8n/n8n-nodes-langchain.informationExtractor",
"position": [
-440,
140
],
"parameters": {
"text": "=You need to extract the review from the following HTML: {{ $json.recensione }}",
"options": {
"systemPromptTemplate": "You are a review expert. You need to extract only the required information and report it without changing anything.\nAll the required information is in the text."
},
"attributes": {
"attributes": [
{
"name": "autore",
"required": true,
"description": "Extract the name of the review author"
},
{
"name": "valutazione",
"type": "number",
"required": true,
"description": "Extract the rating given to the review (from 1 to 5)"
},
{
"name": "data",
"required": true,
"description": "Extract review date in YYYY-MM-DD format"
},
{
"name": "titolo",
"required": true,
"description": "Extract the review title"
},
{
"name": "testo",
"required": true,
"description": "Extract the review text"
},
{
"name": "n_recensioni",
"type": "number",
"required": true,
"description": "Extract the total number of reviews made by the user"
},
{
"name": "nazione",
"required": true,
"description": "Extract the country of the user who wrote the review. Must be two characters"
}
]
}
},
"typeVersion": 1
},
{
"id": "0036f3b1-4832-4a35-8694-0893475a4119",
"name": "If",
"type": "n8n-nodes-base.if",
"position": [
60,
-100
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "loose"
},
"combinator": "and",
"conditions": [
{
"id": "ab666549-4eec-40e2-a702-0575c094a2d4",
"operator": {
"type": "string",
"operation": "empty",
"singleValue": true
},
"leftValue": "={{ $json.Valutazione }}",
"rightValue": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}"
}
]
},
"looseTypeValidation": true
},
"executeOnce": false,
"typeVersion": 2.2
},
{
"id": "5423b55d-eb6c-41c6-9b26-410e3c92b85d",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-700,
-380
],
"parameters": {},
"typeVersion": 1
},
{
"id": "506cdaa1-e0ba-4f29-b137-69d321b13c94",
"name": "Limit1",
"type": "n8n-nodes-base.limit",
"position": [
540,
-380
],
"parameters": {
"maxItems": 3
},
"typeVersion": 1
},
{
"id": "40f1e30d-8aed-4995-b4e4-2239248bd6e7",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-460,
-480
],
"parameters": {
"width": 212.25249169435213,
"height": 245.55481727574733,
"content": "Change to the name of the company registered on Trustpilot and the maximum number of pages to scrape"
},
"typeVersion": 1
},
{
"id": "e6d2fec1-7255-4270-86b4-6d6f39f44ccb",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-460,
80
],
"parameters": {
"width": 381,
"height": 177,
"content": "Extract all information with DeepSeek (remember to change base_url with https://api.deepseek.com/v1)"
},
"typeVersion": 1
},
{
"id": "af5e962c-4faf-41cc-a8b8-2fbb145b7af6",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-160
],
"parameters": {
"width": 501.28903654485043,
"height": 195.84053156146172,
"content": "Check if the review has already been saved to Google Drive"
},
"typeVersion": 1
},
{
"id": "400dff0c-8b2e-4fe2-933e-1f4d14624ca1",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
80
],
"parameters": {
"width": 301.27574750830576,
"height": 177.34219269102988,
"content": "Analyze review sentiment"
},
"typeVersion": 1
},
{
"id": "52757ade-4206-40f9-bf4f-c3aefb004d2e",
"name": "Set Parameters",
"type": "n8n-nodes-base.set",
"position": [
-440,
-380
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "556e201d-242a-4c0e-bc13-787c2b60f800",
"name": "company_id",
"type": "string",
"value": "COMPANY"
},
{
"id": "a1f239df-df08-41d8-8b78-d6502266a581",
"name": "max_page",
"type": "number",
"value": 2
}
]
}
},
"typeVersion": 3.4
},
{
"id": "cd7e9d36-7ecd-4d9c-b552-8f46b0cfcc03",
"name": "Get reviews",
"type": "n8n-nodes-base.httpRequest",
"position": [
-200,
-380
],
"parameters": {
"url": "=https://it.trustpilot.com/review/{{ $json.company_id }}",
"options": {
"pagination": {
"pagination": {
"parameters": {
"parameters": [
{
"name": "page",
"value": "={{ $pageCount + 1 }}"
}
]
},
"maxRequests": "={{ $json.max_page }}",
"requestInterval": 5000,
"limitPagesFetched": true
}
}
},
"sendQuery": true,
"queryParameters": {
"parameters": [
{
"name": "sort",
"value": "recency"
}
]
}
},
"typeVersion": 4.2
},
{
"id": "476ff7b6-ab30-4674-a7fe-b032128ee51a",
"name": "Extract",
"type": "n8n-nodes-base.html",
"position": [
60,
-380
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "recensioni",
"attribute": "href",
"cssSelector": "article section a",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "a2a35455-7d3e-4c4c-aa66-6cbbd48d867a",
"name": "Get rows",
"type": "n8n-nodes-base.googleSheets",
"position": [
-200,
-100
],
"parameters": {
"options": {},
"filtersUI": {
"values": [
{
"lookupValue": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}",
"lookupColumn": "Id"
}
]
},
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Review"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"typeVersion": 4.5
},
{
"id": "2d507fe6-a4fc-42ff-97ff-dfd552c651ab",
"name": "Get Google Sheets",
"type": "n8n-nodes-base.googleSheets",
"position": [
-440,
-100
],
"parameters": {
"columns": {
"value": {
"Id": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}"
},
"schema": [
{
"id": "Id",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Id",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Data",
"type": "string",
"display": true,
"required": false,
"displayName": "Data",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Nome",
"type": "string",
"display": true,
"required": false,
"displayName": "Nome",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Titolo",
"type": "string",
"display": true,
"required": false,
"displayName": "Titolo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Testo",
"type": "string",
"display": true,
"required": false,
"displayName": "Testo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Località",
"type": "string",
"display": true,
"required": false,
"displayName": "Località",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "N. Recensioni",
"type": "string",
"display": true,
"required": false,
"displayName": "N. Recensioni",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "URL",
"type": "string",
"display": true,
"required": false,
"displayName": "URL",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Valutazione",
"type": "string",
"display": true,
"required": false,
"displayName": "Valutazione",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Sentiment",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Sentiment",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"Id"
],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "appendOrUpdate",
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Reviews"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"executeOnce": false,
"typeVersion": 4.5
},
{
"id": "0a1fab6e-96b7-403b-884e-f67be6e23fa5",
"name": "Get Single review",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
-120
],
"parameters": {
"url": "=https://it.trustpilot.com{{ $('Split Out').item.json.recensioni }}",
"options": {}
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "7d322d76-1032-405a-9d46-2958761a184d",
"name": "Extract review",
"type": "n8n-nodes-base.html",
"position": [
540,
-120
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "recensione",
"cssSelector": "article",
"returnArray": true
}
]
}
},
"typeVersion": 1.2
},
{
"id": "952484e5-8e87-4eb3-99a6-5bf26c701ba8",
"name": "Update sheet",
"type": "n8n-nodes-base.googleSheets",
"position": [
520,
120
],
"parameters": {
"columns": {
"value": {
"Id": "={{ $('Split Out').item.json.recensioni.replace('/reviews/','') }}",
"URL": "=https://it.trustpilot.com{{ $('Split Out').item.json.recensioni }}",
"Data": "={{ $('Information Extractor').item.json.output.data }}",
"Nome": "={{ $json.output.autore }}",
"Testo": "={{ $('Information Extractor').item.json.output.testo }}",
"Titolo": "={{ $('Information Extractor').item.json.output.titolo }}",
"Località": "={{ $('Information Extractor').item.json.output.nazione }}",
"Sentiment": "={{ $json.sentimentAnalysis.category }}",
"Valutazione": "={{ $('Information Extractor').item.json.output.valutazione }}",
"N. Recensioni": "={{ $('Information Extractor').item.json.output.n_recensioni }}"
},
"schema": [
{
"id": "Id",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Id",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Data",
"type": "string",
"display": true,
"required": false,
"displayName": "Data",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Nome",
"type": "string",
"display": true,
"required": false,
"displayName": "Nome",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Titolo",
"type": "string",
"display": true,
"required": false,
"displayName": "Titolo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Testo",
"type": "string",
"display": true,
"required": false,
"displayName": "Testo",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Località",
"type": "string",
"display": true,
"required": false,
"displayName": "Località",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "N. Recensioni",
"type": "string",
"display": true,
"required": false,
"displayName": "N. Recensioni",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "URL",
"type": "string",
"display": true,
"required": false,
"displayName": "URL",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Valutazione",
"type": "string",
"display": true,
"required": false,
"displayName": "Valutazione",
"defaultMatch": false,
"canBeUsedToMatch": true
},
{
"id": "Sentiment",
"type": "string",
"display": true,
"removed": false,
"required": false,
"displayName": "Sentiment",
"defaultMatch": false,
"canBeUsedToMatch": true
}
],
"mappingMode": "defineBelow",
"matchingColumns": [
"Id"
],
"attemptToConvertTypes": false,
"convertFieldsToString": false
},
"options": {},
"operation": "appendOrUpdate",
"sheetName": {
"__rl": true,
"mode": "list",
"value": "gid=0",
"cachedResultUrl": "",
"cachedResultName": "Foglio1"
},
"documentId": {
"__rl": true,
"mode": "list",
"value": "1QZhQqg79-HVBQh8Y2ihMq67UIYIRrJFKLQalcFvtDaY",
"cachedResultUrl": "",
"cachedResultName": "Trustpilot Reviews"
}
},
"credentials": {
"googleSheetsOAuth2Api": {
"id": "JYR6a64Qecd6t8Hb",
"name": "Google Sheets account"
}
},
"typeVersion": 4.5
},
{
"id": "eb853885-816d-4df7-b5ac-900fa89d3df9",
"name": "Sentiment Analysis",
"type": "@n8n/n8n-nodes-langchain.sentimentAnalysis",
"position": [
60,
140
],
"parameters": {
"options": {
"categories": "Positive, Neutral, Negative",
"systemPromptTemplate": "You are highly intelligent and accurate sentiment analyzer. Analyze the sentiment of the provided text. Categorize it into one of the following: {categories}. Use the provided formatting instructions. Only output the JSON."
},
"inputText": "={{ $json.output.testo }}"
},
"typeVersion": 1
},
{
"id": "79f1b9ea-6297-4735-9c0f-9f28dd65efa0",
"name": "DeepSeek Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
-460,
320
],
"parameters": {
"model": "deepseek-reasoner",
"options": {
"baseURL": "https://api.deepseek.com/v1"
}
},
"credentials": {
"openAiApi": {
"id": "97Cz4cqyiy1RdcQL",
"name": "DeepSeek"
}
},
"typeVersion": 1
},
{
"id": "159cc88e-1dd3-4bba-a3c8-59a9aad14c88",
"name": "OpenAI Chat Model",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
40,
320
],
"parameters": {
"options": {}
},
"credentials": {
"openAiApi": {
"id": "CDX6QM4gLYanh0P4",
"name": "OpenAi account"
}
},
"typeVersion": 1.1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "43c8ee74-159c-4217-9cb4-554c63a3b183",
"connections": {
"If": {
"main": [
[
{
"node": "Get Single review",
"type": "main",
"index": 0
}
]
]
},
"Limit1": {
"main": [
[
{
"node": "Get Google Sheets",
"type": "main",
"index": 0
}
]
]
},
"Extract": {
"main": [
[
{
"node": "Split Out",
"type": "main",
"index": 0
}
]
]
},
"Get rows": {
"main": [
[
{
"node": "If",
"type": "main",
"index": 0
}
]
]
},
"Split Out": {
"main": [
[
{
"node": "Limit1",
"type": "main",
"index": 0
}
]
]
},
"Get reviews": {
"main": [
[
{
"node": "Extract",
"type": "main",
"index": 0
}
]
]
},
"Extract review": {
"main": [
[
{
"node": "Information Extractor",
"type": "main",
"index": 0
}
]
]
},
"Set Parameters": {
"main": [
[
{
"node": "Get reviews",
"type": "main",
"index": 0
}
]
]
},
"Get Google Sheets": {
"main": [
[
{
"node": "Get rows",
"type": "main",
"index": 0
}
]
]
},
"Get Single review": {
"main": [
[
{
"node": "Extract review",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model": {
"ai_languageModel": [
[
{
"node": "Sentiment Analysis",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Sentiment Analysis": {
"main": [
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
],
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
],
[
{
"node": "Update sheet",
"type": "main",
"index": 0
}
]
]
},
"DeepSeek Chat Model": {
"ai_languageModel": [
[
{
"node": "Information Extractor",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Information Extractor": {
"main": [
[
{
"node": "Sentiment Analysis",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Set Parameters",
"type": "main",
"index": 0
}
]
]
}
}
}

885
AI_Research_RAG_and_Data_Analysis/Scrape and summarize posts of a news site without RSS feed using AI and save them to a NocoDB.json Normal file
View File

@@ -0,0 +1,885 @@
{
"id": "xM8Z5vZVNTNjCySL",
"meta": {
"instanceId": "b8ef33547995f2a520f12118ac1f7819ea58faa7a1096148cac519fa08be8e99"
},
"name": "News Extraction",
"tags": [],
"nodes": [
{
"id": "97711d12-20de-40aa-b994-d2b10f20a5e5",
"name": "Extract the HTML with the right css class",
"type": "n8n-nodes-base.html",
"position": [
-500,
0
],
"parameters": {
"options": {
"trimValues": true
},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"attribute": "href",
"cssSelector": "=div:nth-child(9) > div:nth-child(3) > a:nth-child(2)",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1
},
{
"id": "b874b570-daae-4878-b525-07ac30756eb1",
"name": "Summary",
"type": "n8n-nodes-base.openAi",
"position": [
-880,
440
],
"parameters": {
"model": "gpt-4-1106-preview",
"prompt": {
"messages": [
{
"content": "=Create a summary in less than 70 words {{ $json[\"content\"] }}"
}
]
},
"options": {},
"resource": "chat"
},
"credentials": {
"openAiApi": {
"id": "0Vdk5RlVe7AoUdAM",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "72696278-2d44-4073-936a-6fe9df1bc7d8",
"name": "Keywords",
"type": "n8n-nodes-base.openAi",
"position": [
-880,
620
],
"parameters": {
"model": "gpt-4-1106-preview",
"prompt": {
"messages": [
{
"content": "=name the 3 most important technical keywords in {{ $json[\"content\"] }} ? just name them without any explanations or other sentences"
}
]
},
"options": {},
"resource": "chat"
},
"credentials": {
"openAiApi": {
"id": "0Vdk5RlVe7AoUdAM",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "0bfdb3be-76ef-4bb3-902f-f0869342b83c",
"name": "Rename keywords",
"type": "n8n-nodes-base.set",
"position": [
-700,
620
],
"parameters": {
"fields": {
"values": [
{
"name": "keywords",
"stringValue": "={{ $json[\"message\"][\"content\"] }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3.1
},
{
"id": "0387cf34-41c9-4729-8570-1db7b17c42f4",
"name": "Rename Summary",
"type": "n8n-nodes-base.set",
"position": [
-700,
440
],
"parameters": {
"fields": {
"values": [
{
"name": "=summary",
"stringValue": "={{ $json[\"message\"][\"content\"] }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3.1
},
{
"id": "5fa1702c-f0bf-4524-bc8f-6f550dd83f1e",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
-480,
560
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "25128a71-b0d5-49a4-adb8-c3fbe03c0a85",
"name": "Extract date",
"type": "n8n-nodes-base.html",
"position": [
-500,
-160
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"cssSelector": "div:nth-child(9) > div:nth-child(2) > span:nth-child(1)",
"returnArray": true
}
]
}
},
"typeVersion": 1
},
{
"id": "138b3bd6-494a-49b9-b5b8-c9febcfef9fb",
"name": "Select posts of last 7 days",
"type": "n8n-nodes-base.code",
"position": [
120,
0
],
"parameters": {
"jsCode": "const currentDate = new Date();\nconst sevenDaysAgo = new Date(currentDate.setDate(currentDate.getDate() - 70)); // Change the number of days going back to your liking (e.g. from -7 to -1) -> BUT sync with the cron job (first node)\n\nconst filteredItems = items.filter(item => {\n const postDate = new Date(item.json[\"Date\"]); // Assuming \"Date\" is the field name in the extracted html\n return postDate >= sevenDaysAgo;\n});\n\nreturn filteredItems;\n"
},
"typeVersion": 2
},
{
"id": "1ace953b-e298-4fc2-8970-327f736889ec",
"name": "Merge date & links",
"type": "n8n-nodes-base.merge",
"position": [
-100,
0
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "bba692fc-c225-41be-a969-179d8b99c071",
"name": "HTTP Request1",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
0
],
"parameters": {
"url": "={{ $json[\"Link\"] }}",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "26671065-631f-4684-9ee1-15f26b4cf1e4",
"name": "Merge Content with Date & Link",
"type": "n8n-nodes-base.merge",
"position": [
500,
260
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "79beb744-97b8-4072-824a-6736b0a080ef",
"name": "Extract individual posts",
"type": "n8n-nodes-base.html",
"position": [
500,
0
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "title",
"cssSelector": "h1.fl-heading > span:nth-child(1)"
},
{
"key": "content",
"cssSelector": ".fl-node-5c7574ae7d5c6 > div:nth-child(1)"
}
]
}
},
"typeVersion": 1
},
{
"id": "e89d9de5-875b-453e-825a-26f2bebcc8df",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
80,
-107
],
"parameters": {
"width": 180.9747474601832,
"height": 276.31054308676767,
"content": "Select only the newest news: todays date going back xy days"
},
"typeVersion": 1
},
{
"id": "8a603f2f-4208-48c7-b169-e5613f13fa7d",
"name": "Merge ChatGPT output with Date & Link",
"type": "n8n-nodes-base.merge",
"position": [
-180,
560
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "e1036421-9ce1-4121-a692-602410ec7c95",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
-539.7802584556148,
-4.722020203185366
],
"parameters": {
"width": 182.2748213508401,
"height": 304.2550759710132,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nExtracting the individual links of the press release page in order to retrieve the individual posts on their respective **url**"
},
"typeVersion": 1
},
{
"id": "3655ab22-6a17-429a-9d9b-d96bbcc78fee",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-538.404803912782,
-304
],
"parameters": {
"width": 178.75185894039254,
"height": 289.463147786618,
"content": "Extracting the dates of the posts of the press release page.\nThe right CSS selector has to be chosen.\n[More info on datagrab.io](https://datagrab.io/blog/guide-to-css-selectors-for-web-scraping/)"
},
"typeVersion": 1
},
{
"id": "2e27fb4c-426a-41e1-b5fb-9b2d78acd2a7",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1300,
-299.82161760751774
],
"parameters": {
"width": 334.4404040637068,
"height": 1127.2017245821128,
"content": "# Scraping posts of a news site without RSS feed\n\n\nThe [News Site](https://www.colt.net/resources/type/news/) from Colt, a telecom company, does not offer an RSS feed, therefore web scraping is the \nchoice to extract and process the news.\n\nThe goal is to get only the newest posts, a summary of each post and their respective (technical) keywords.\n\nNote that the news site offers the links to each news post, but not the individual news. We collect first the links and dates of each post before extracting the newest ones.\n\nThe result is sent to a SQL database, in this case a NocoDB database.\n\nThis process happens each week thru a cron job.\n\n**Requirements**:\n- Basic understanding of CSS selectors and how to get them via browser (usually: right click &rarr; inspect)\n- ChatGPT API account - normal account is not sufficient\n- A NocoDB database - of course you may choose any type of output target\n\n**Assumptions**:\n- CSS selectors work on the news site\n- The post has a date with own CSS selector - meaning date is not part of the news content\n\n**\"Warnings\"**\n- Not every site likes to be scraped, especially not in high frequency\n- Each website is structured in different ways, the workflow may then need several adaptations.\n\n\nHappy about any suggestion to improve. You may contact me on **Mastodon**: https://bonn.social/@askans"
},
"typeVersion": 1
},
{
"id": "d43bd5b7-2aff-4a07-8aca-ca4747ec6c4d",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-927.8447474890202,
-80
],
"parameters": {
"width": 153.90180146729315,
"height": 237.91333335255808,
"content": "Weekly cron job"
},
"typeVersion": 1
},
{
"id": "e732d136-fcf1-4fc3-8bb6-bdcea3c78d9e",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
-760,
-80
],
"parameters": {
"width": 185.41515152389002,
"height": 241.454848504947,
"content": "The html of the news site is being retrieved: https://www.colt.net/resources/type/news/"
},
"typeVersion": 1
},
{
"id": "d5e29ec3-5ef2-42f3-b316-9350644dbba4",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
-306
],
"parameters": {
"width": 187.3613302133812,
"height": 469.2923233086395,
"content": "As the extraction are returned as arrays, they transformed into individual JSON items to enable looping with other nodes"
},
"typeVersion": 1
},
{
"id": "1af15c45-32c0-4abf-a35d-be7206823569",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
-120,
-103.54151515238902
],
"parameters": {
"width": 150,
"height": 274.50898992724416,
"content": "The links of the individual posts and the dates of the posts "
},
"typeVersion": 1
},
{
"id": "f7c42748-f227-42d0-a9e2-fcb16dbd0f75",
"name": "Retrieve the web page for further processsing",
"type": "n8n-nodes-base.httpRequest",
"position": [
-720,
0
],
"parameters": {
"url": "https://www.colt.net/resources/type/news/",
"options": {
"response": {
"response": {
"responseFormat": "text"
}
}
}
},
"typeVersion": 4.1
},
{
"id": "b2c36f26-8221-478f-a4b0-22758b1e5e58",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
292,
-100
],
"parameters": {
"width": 155.0036363426638,
"height": 272.1479798256519,
"content": "Get the html of each individual **newest** post"
},
"typeVersion": 1
},
{
"id": "6ae05c31-c09a-4b4e-a013-41571937bc39",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
-100
],
"parameters": {
"width": 184.07417896879767,
"height": 269.2504410842093,
"content": "Extracting the title & content (text) of each individual news post with the right CSS selector"
},
"typeVersion": 1
},
{
"id": "e2da76d4-0c8c-4c61-924f-50aa9387e9ab",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
180
],
"parameters": {
"width": 191.87778190338406,
"height": 234.13422787857044,
"content": "Merge link to url, date with content (text) and title of each news psot"
},
"typeVersion": 1
},
{
"id": "c124aaac-dce6-4658-9027-bdfe5c0c81e6",
"name": "Sticky Note12",
"type": "n8n-nodes-base.stickyNote",
"position": [
-907.2264215202996,
331.0681740778203
],
"parameters": {
"width": 150,
"height": 256.2444361932317,
"content": "Create a summary of each news post with ChatGPT. You need a ChatGPT API account for this"
},
"typeVersion": 1
},
{
"id": "c9037e74-007b-4e44-b7f9-90e78b853eb5",
"name": "Sticky Note13",
"type": "n8n-nodes-base.stickyNote",
"position": [
-909.595196087218,
610.7495589157902
],
"parameters": {
"width": 152.85976723045226,
"height": 218.52702200939785,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\nGet the 3 keywords of each news post"
},
"typeVersion": 1
},
{
"id": "756397d9-de80-4114-9dee-b4f4b9593333",
"name": "Sticky Note14",
"type": "n8n-nodes-base.stickyNote",
"position": [
-740,
340
],
"parameters": {
"width": 182.7735784797001,
"height": 489.05192374172555,
"content": "Just a renaming of data fields and eliminating unnecessary ones"
},
"typeVersion": 1
},
{
"id": "a0dcb254-f064-45ed-8e22-30a6d079085b",
"name": "Sticky Note15",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
480
],
"parameters": {
"width": 169.7675735887227,
"height": 254.94383570413422,
"content": "Merge summary and keywords of each news post"
},
"typeVersion": 1
},
{
"id": "82993166-b273-4b82-a954-554c6892f825",
"name": "Schedule Trigger each week",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
-900,
0
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
3
],
"triggerAtHour": 4,
"triggerAtMinute": 32
}
]
}
},
"typeVersion": 1.1
},
{
"id": "3d670eb9-5a36-4cd9-8d2c-40adf848485e",
"name": "Sticky Note16",
"type": "n8n-nodes-base.stickyNote",
"position": [
-220,
477.5081090810816
],
"parameters": {
"width": 180.1723775015045,
"height": 260.5279202647822,
"content": "Add title, link and date to summary and keywords of each news post"
},
"typeVersion": 1
},
{
"id": "62021393-e988-4834-9fa2-75a957b42890",
"name": "NocoDB news database",
"type": "n8n-nodes-base.nocoDb",
"position": [
60,
560
],
"parameters": {
"table": "mhbalmu9aaqcun6",
"fieldsUi": {
"fieldValues": [
{
"fieldName": "=News_Source",
"fieldValue": "=Colt"
},
{
"fieldName": "Title",
"fieldValue": "={{ $json[\"title\"] }}"
},
{
"fieldName": "Date",
"fieldValue": "={{ $json[\"Date\"] }}"
},
{
"fieldName": "Link",
"fieldValue": "={{ $json[\"Link\"] }}"
},
{
"fieldName": "Summary",
"fieldValue": "={{ $json[\"summary\"] }}"
},
{
"fieldName": "Keywords",
"fieldValue": "={{ $json[\"keywords\"] }}"
}
]
},
"operation": "create",
"projectId": "prqu4e8bjj4bv1j",
"authentication": "nocoDbApiToken"
},
"credentials": {
"nocoDbApiToken": {
"id": "gjNns0VJMS3P2RQ3",
"name": "NocoDB Token account"
}
},
"typeVersion": 2
},
{
"id": "e59e9fab-10a7-470b-afa6-e1d4b4e57723",
"name": "Sticky Note17",
"type": "n8n-nodes-base.stickyNote",
"position": [
280,
480
],
"parameters": {
"width": 483.95825869942666,
"height": 268.5678114630957,
"content": "## News summaries and keywords &rarr; database\n\n[NocoDB](https://nocodb.com/) is an SQL database, here we store the news summaries and keywords for further processing. Any other output target can be chosen here, e.g. e-mail, Excel etc.\n\nYou need first have that database structured before appending the news summaries and additional fields. The you can shape this node.\n\nSome fields may be edited in the database itself (e.g. relevance of the news to you) and may be filled therefore with a default value or not at all"
},
"typeVersion": 1
},
{
"id": "253b414b-9a5b-4a25-892b-9aa011d55d28",
"name": "Sticky Note18",
"type": "n8n-nodes-base.stickyNote",
"position": [
20,
480
],
"parameters": {
"width": 262.99083066277313,
"height": 268.56781146309544,
"content": ""
},
"typeVersion": 1
},
{
"id": "438e8dde-ce0a-4e5e-8d62-d735d19ec189",
"name": "Create single link items",
"type": "n8n-nodes-base.itemLists",
"position": [
-300,
0
],
"parameters": {
"options": {
"destinationFieldName": "Link"
},
"fieldToSplitOut": "data"
},
"typeVersion": 3
},
{
"id": "d721776b-fefc-4e72-91ef-6710f10b0393",
"name": "Create single date items",
"type": "n8n-nodes-base.itemLists",
"position": [
-300,
-160
],
"parameters": {
"options": {
"destinationFieldName": "Date"
},
"fieldToSplitOut": "data"
},
"typeVersion": 3
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ff89d802-3bcf-4b34-9cd9-776b1f3b5eab",
"connections": {
"Merge": {
"main": [
[
{
"node": "Merge ChatGPT output with Date & Link",
"type": "main",
"index": 1
}
]
]
},
"Summary": {
"main": [
[
{
"node": "Rename Summary",
"type": "main",
"index": 0
}
]
]
},
"Keywords": {
"main": [
[
{
"node": "Rename keywords",
"type": "main",
"index": 0
}
]
]
},
"Extract date": {
"main": [
[
{
"node": "Create single date items",
"type": "main",
"index": 0
}
]
]
},
"HTTP Request1": {
"main": [
[
{
"node": "Extract individual posts",
"type": "main",
"index": 0
}
]
]
},
"Rename Summary": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Rename keywords": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Merge date & links": {
"main": [
[
{
"node": "Select posts of last 7 days",
"type": "main",
"index": 0
}
]
]
},
"Create single date items": {
"main": [
[
{
"node": "Merge date & links",
"type": "main",
"index": 0
}
]
]
},
"Create single link items": {
"main": [
[
{
"node": "Merge date & links",
"type": "main",
"index": 1
}
]
]
},
"Extract individual posts": {
"main": [
[
{
"node": "Merge Content with Date & Link",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger each week": {
"main": [
[
{
"node": "Retrieve the web page for further processsing",
"type": "main",
"index": 0
}
]
]
},
"Select posts of last 7 days": {
"main": [
[
{
"node": "Merge Content with Date & Link",
"type": "main",
"index": 1
},
{
"node": "HTTP Request1",
"type": "main",
"index": 0
}
]
]
},
"Merge Content with Date & Link": {
"main": [
[
{
"node": "Summary",
"type": "main",
"index": 0
},
{
"node": "Keywords",
"type": "main",
"index": 0
},
{
"node": "Merge ChatGPT output with Date & Link",
"type": "main",
"index": 0
}
]
]
},
"Merge ChatGPT output with Date & Link": {
"main": [
[
{
"node": "NocoDB news database",
"type": "main",
"index": 0
}
]
]
},
"Extract the HTML with the right css class": {
"main": [
[
{
"node": "Create single link items",
"type": "main",
"index": 0
}
]
]
},
"Retrieve the web page for further processsing": {
"main": [
[
{
"node": "Extract the HTML with the right css class",
"type": "main",
"index": 0
},
{
"node": "Extract date",
"type": "main",
"index": 0
}
]
]
}
}
}

885
AI_Research_RAG_and_Data_Analysis/Scrape and summarize posts of a news site without RSS feed using AI and save them to a NocoDB.txt
View File

@@ -1,885 +0,0 @@
{
"id": "xM8Z5vZVNTNjCySL",
"meta": {
"instanceId": "b8ef33547995f2a520f12118ac1f7819ea58faa7a1096148cac519fa08be8e99"
},
"name": "News Extraction",
"tags": [],
"nodes": [
{
"id": "97711d12-20de-40aa-b994-d2b10f20a5e5",
"name": "Extract the HTML with the right css class",
"type": "n8n-nodes-base.html",
"position": [
-500,
0
],
"parameters": {
"options": {
"trimValues": true
},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"attribute": "href",
"cssSelector": "=div:nth-child(9) > div:nth-child(3) > a:nth-child(2)",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1
},
{
"id": "b874b570-daae-4878-b525-07ac30756eb1",
"name": "Summary",
"type": "n8n-nodes-base.openAi",
"position": [
-880,
440
],
"parameters": {
"model": "gpt-4-1106-preview",
"prompt": {
"messages": [
{
"content": "=Create a summary in less than 70 words {{ $json[\"content\"] }}"
}
]
},
"options": {},
"resource": "chat"
},
"credentials": {
"openAiApi": {
"id": "0Vdk5RlVe7AoUdAM",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "72696278-2d44-4073-936a-6fe9df1bc7d8",
"name": "Keywords",
"type": "n8n-nodes-base.openAi",
"position": [
-880,
620
],
"parameters": {
"model": "gpt-4-1106-preview",
"prompt": {
"messages": [
{
"content": "=name the 3 most important technical keywords in {{ $json[\"content\"] }} ? just name them without any explanations or other sentences"
}
]
},
"options": {},
"resource": "chat"
},
"credentials": {
"openAiApi": {
"id": "0Vdk5RlVe7AoUdAM",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "0bfdb3be-76ef-4bb3-902f-f0869342b83c",
"name": "Rename keywords",
"type": "n8n-nodes-base.set",
"position": [
-700,
620
],
"parameters": {
"fields": {
"values": [
{
"name": "keywords",
"stringValue": "={{ $json[\"message\"][\"content\"] }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3.1
},
{
"id": "0387cf34-41c9-4729-8570-1db7b17c42f4",
"name": "Rename Summary",
"type": "n8n-nodes-base.set",
"position": [
-700,
440
],
"parameters": {
"fields": {
"values": [
{
"name": "=summary",
"stringValue": "={{ $json[\"message\"][\"content\"] }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3.1
},
{
"id": "5fa1702c-f0bf-4524-bc8f-6f550dd83f1e",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
-480,
560
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "25128a71-b0d5-49a4-adb8-c3fbe03c0a85",
"name": "Extract date",
"type": "n8n-nodes-base.html",
"position": [
-500,
-160
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "data",
"cssSelector": "div:nth-child(9) > div:nth-child(2) > span:nth-child(1)",
"returnArray": true
}
]
}
},
"typeVersion": 1
},
{
"id": "138b3bd6-494a-49b9-b5b8-c9febcfef9fb",
"name": "Select posts of last 7 days",
"type": "n8n-nodes-base.code",
"position": [
120,
0
],
"parameters": {
"jsCode": "const currentDate = new Date();\nconst sevenDaysAgo = new Date(currentDate.setDate(currentDate.getDate() - 70)); // Change the number of days going back to your liking (e.g. from -7 to -1) -> BUT sync with the cron job (first node)\n\nconst filteredItems = items.filter(item => {\n const postDate = new Date(item.json[\"Date\"]); // Assuming \"Date\" is the field name in the extracted html\n return postDate >= sevenDaysAgo;\n});\n\nreturn filteredItems;\n"
},
"typeVersion": 2
},
{
"id": "1ace953b-e298-4fc2-8970-327f736889ec",
"name": "Merge date & links",
"type": "n8n-nodes-base.merge",
"position": [
-100,
0
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "bba692fc-c225-41be-a969-179d8b99c071",
"name": "HTTP Request1",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
0
],
"parameters": {
"url": "={{ $json[\"Link\"] }}",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "26671065-631f-4684-9ee1-15f26b4cf1e4",
"name": "Merge Content with Date & Link",
"type": "n8n-nodes-base.merge",
"position": [
500,
260
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "79beb744-97b8-4072-824a-6736b0a080ef",
"name": "Extract individual posts",
"type": "n8n-nodes-base.html",
"position": [
500,
0
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "title",
"cssSelector": "h1.fl-heading > span:nth-child(1)"
},
{
"key": "content",
"cssSelector": ".fl-node-5c7574ae7d5c6 > div:nth-child(1)"
}
]
}
},
"typeVersion": 1
},
{
"id": "e89d9de5-875b-453e-825a-26f2bebcc8df",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
80,
-107
],
"parameters": {
"width": 180.9747474601832,
"height": 276.31054308676767,
"content": "Select only the newest news: todays date going back xy days"
},
"typeVersion": 1
},
{
"id": "8a603f2f-4208-48c7-b169-e5613f13fa7d",
"name": "Merge ChatGPT output with Date & Link",
"type": "n8n-nodes-base.merge",
"position": [
-180,
560
],
"parameters": {
"mode": "combine",
"options": {},
"combinationMode": "mergeByPosition"
},
"typeVersion": 2.1
},
{
"id": "e1036421-9ce1-4121-a692-602410ec7c95",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"disabled": true,
"position": [
-539.7802584556148,
-4.722020203185366
],
"parameters": {
"width": 182.2748213508401,
"height": 304.2550759710132,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nExtracting the individual links of the press release page in order to retrieve the individual posts on their respective **url**"
},
"typeVersion": 1
},
{
"id": "3655ab22-6a17-429a-9d9b-d96bbcc78fee",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-538.404803912782,
-304
],
"parameters": {
"width": 178.75185894039254,
"height": 289.463147786618,
"content": "Extracting the dates of the posts of the press release page.\nThe right CSS selector has to be chosen.\n[More info on datagrab.io](https://datagrab.io/blog/guide-to-css-selectors-for-web-scraping/)"
},
"typeVersion": 1
},
{
"id": "2e27fb4c-426a-41e1-b5fb-9b2d78acd2a7",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1300,
-299.82161760751774
],
"parameters": {
"width": 334.4404040637068,
"height": 1127.2017245821128,
"content": "# Scraping posts of a news site without RSS feed\n\n\nThe [News Site](https://www.colt.net/resources/type/news/) from Colt, a telecom company, does not offer an RSS feed, therefore web scraping is the \nchoice to extract and process the news.\n\nThe goal is to get only the newest posts, a summary of each post and their respective (technical) keywords.\n\nNote that the news site offers the links to each news post, but not the individual news. We collect first the links and dates of each post before extracting the newest ones.\n\nThe result is sent to a SQL database, in this case a NocoDB database.\n\nThis process happens each week thru a cron job.\n\n**Requirements**:\n- Basic understanding of CSS selectors and how to get them via browser (usually: right click &rarr; inspect)\n- ChatGPT API account - normal account is not sufficient\n- A NocoDB database - of course you may choose any type of output target\n\n**Assumptions**:\n- CSS selectors work on the news site\n- The post has a date with own CSS selector - meaning date is not part of the news content\n\n**\"Warnings\"**\n- Not every site likes to be scraped, especially not in high frequency\n- Each website is structured in different ways, the workflow may then need several adaptations.\n\n\nHappy about any suggestion to improve. You may contact me on **Mastodon**: https://bonn.social/@askans"
},
"typeVersion": 1
},
{
"id": "d43bd5b7-2aff-4a07-8aca-ca4747ec6c4d",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-927.8447474890202,
-80
],
"parameters": {
"width": 153.90180146729315,
"height": 237.91333335255808,
"content": "Weekly cron job"
},
"typeVersion": 1
},
{
"id": "e732d136-fcf1-4fc3-8bb6-bdcea3c78d9e",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
-760,
-80
],
"parameters": {
"width": 185.41515152389002,
"height": 241.454848504947,
"content": "The html of the news site is being retrieved: https://www.colt.net/resources/type/news/"
},
"typeVersion": 1
},
{
"id": "d5e29ec3-5ef2-42f3-b316-9350644dbba4",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
-306
],
"parameters": {
"width": 187.3613302133812,
"height": 469.2923233086395,
"content": "As the extraction are returned as arrays, they transformed into individual JSON items to enable looping with other nodes"
},
"typeVersion": 1
},
{
"id": "1af15c45-32c0-4abf-a35d-be7206823569",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
-120,
-103.54151515238902
],
"parameters": {
"width": 150,
"height": 274.50898992724416,
"content": "The links of the individual posts and the dates of the posts "
},
"typeVersion": 1
},
{
"id": "f7c42748-f227-42d0-a9e2-fcb16dbd0f75",
"name": "Retrieve the web page for further processsing",
"type": "n8n-nodes-base.httpRequest",
"position": [
-720,
0
],
"parameters": {
"url": "https://www.colt.net/resources/type/news/",
"options": {
"response": {
"response": {
"responseFormat": "text"
}
}
}
},
"typeVersion": 4.1
},
{
"id": "b2c36f26-8221-478f-a4b0-22758b1e5e58",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
292,
-100
],
"parameters": {
"width": 155.0036363426638,
"height": 272.1479798256519,
"content": "Get the html of each individual **newest** post"
},
"typeVersion": 1
},
{
"id": "6ae05c31-c09a-4b4e-a013-41571937bc39",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
-100
],
"parameters": {
"width": 184.07417896879767,
"height": 269.2504410842093,
"content": "Extracting the title & content (text) of each individual news post with the right CSS selector"
},
"typeVersion": 1
},
{
"id": "e2da76d4-0c8c-4c61-924f-50aa9387e9ab",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
180
],
"parameters": {
"width": 191.87778190338406,
"height": 234.13422787857044,
"content": "Merge link to url, date with content (text) and title of each news psot"
},
"typeVersion": 1
},
{
"id": "c124aaac-dce6-4658-9027-bdfe5c0c81e6",
"name": "Sticky Note12",
"type": "n8n-nodes-base.stickyNote",
"position": [
-907.2264215202996,
331.0681740778203
],
"parameters": {
"width": 150,
"height": 256.2444361932317,
"content": "Create a summary of each news post with ChatGPT. You need a ChatGPT API account for this"
},
"typeVersion": 1
},
{
"id": "c9037e74-007b-4e44-b7f9-90e78b853eb5",
"name": "Sticky Note13",
"type": "n8n-nodes-base.stickyNote",
"position": [
-909.595196087218,
610.7495589157902
],
"parameters": {
"width": 152.85976723045226,
"height": 218.52702200939785,
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\nGet the 3 keywords of each news post"
},
"typeVersion": 1
},
{
"id": "756397d9-de80-4114-9dee-b4f4b9593333",
"name": "Sticky Note14",
"type": "n8n-nodes-base.stickyNote",
"position": [
-740,
340
],
"parameters": {
"width": 182.7735784797001,
"height": 489.05192374172555,
"content": "Just a renaming of data fields and eliminating unnecessary ones"
},
"typeVersion": 1
},
{
"id": "a0dcb254-f064-45ed-8e22-30a6d079085b",
"name": "Sticky Note15",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
480
],
"parameters": {
"width": 169.7675735887227,
"height": 254.94383570413422,
"content": "Merge summary and keywords of each news post"
},
"typeVersion": 1
},
{
"id": "82993166-b273-4b82-a954-554c6892f825",
"name": "Schedule Trigger each week",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
-900,
0
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
3
],
"triggerAtHour": 4,
"triggerAtMinute": 32
}
]
}
},
"typeVersion": 1.1
},
{
"id": "3d670eb9-5a36-4cd9-8d2c-40adf848485e",
"name": "Sticky Note16",
"type": "n8n-nodes-base.stickyNote",
"position": [
-220,
477.5081090810816
],
"parameters": {
"width": 180.1723775015045,
"height": 260.5279202647822,
"content": "Add title, link and date to summary and keywords of each news post"
},
"typeVersion": 1
},
{
"id": "62021393-e988-4834-9fa2-75a957b42890",
"name": "NocoDB news database",
"type": "n8n-nodes-base.nocoDb",
"position": [
60,
560
],
"parameters": {
"table": "mhbalmu9aaqcun6",
"fieldsUi": {
"fieldValues": [
{
"fieldName": "=News_Source",
"fieldValue": "=Colt"
},
{
"fieldName": "Title",
"fieldValue": "={{ $json[\"title\"] }}"
},
{
"fieldName": "Date",
"fieldValue": "={{ $json[\"Date\"] }}"
},
{
"fieldName": "Link",
"fieldValue": "={{ $json[\"Link\"] }}"
},
{
"fieldName": "Summary",
"fieldValue": "={{ $json[\"summary\"] }}"
},
{
"fieldName": "Keywords",
"fieldValue": "={{ $json[\"keywords\"] }}"
}
]
},
"operation": "create",
"projectId": "prqu4e8bjj4bv1j",
"authentication": "nocoDbApiToken"
},
"credentials": {
"nocoDbApiToken": {
"id": "gjNns0VJMS3P2RQ3",
"name": "NocoDB Token account"
}
},
"typeVersion": 2
},
{
"id": "e59e9fab-10a7-470b-afa6-e1d4b4e57723",
"name": "Sticky Note17",
"type": "n8n-nodes-base.stickyNote",
"position": [
280,
480
],
"parameters": {
"width": 483.95825869942666,
"height": 268.5678114630957,
"content": "## News summaries and keywords &rarr; database\n\n[NocoDB](https://nocodb.com/) is an SQL database, here we store the news summaries and keywords for further processing. Any other output target can be chosen here, e.g. e-mail, Excel etc.\n\nYou need first have that database structured before appending the news summaries and additional fields. The you can shape this node.\n\nSome fields may be edited in the database itself (e.g. relevance of the news to you) and may be filled therefore with a default value or not at all"
},
"typeVersion": 1
},
{
"id": "253b414b-9a5b-4a25-892b-9aa011d55d28",
"name": "Sticky Note18",
"type": "n8n-nodes-base.stickyNote",
"position": [
20,
480
],
"parameters": {
"width": 262.99083066277313,
"height": 268.56781146309544,
"content": ""
},
"typeVersion": 1
},
{
"id": "438e8dde-ce0a-4e5e-8d62-d735d19ec189",
"name": "Create single link items",
"type": "n8n-nodes-base.itemLists",
"position": [
-300,
0
],
"parameters": {
"options": {
"destinationFieldName": "Link"
},
"fieldToSplitOut": "data"
},
"typeVersion": 3
},
{
"id": "d721776b-fefc-4e72-91ef-6710f10b0393",
"name": "Create single date items",
"type": "n8n-nodes-base.itemLists",
"position": [
-300,
-160
],
"parameters": {
"options": {
"destinationFieldName": "Date"
},
"fieldToSplitOut": "data"
},
"typeVersion": 3
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ff89d802-3bcf-4b34-9cd9-776b1f3b5eab",
"connections": {
"Merge": {
"main": [
[
{
"node": "Merge ChatGPT output with Date & Link",
"type": "main",
"index": 1
}
]
]
},
"Summary": {
"main": [
[
{
"node": "Rename Summary",
"type": "main",
"index": 0
}
]
]
},
"Keywords": {
"main": [
[
{
"node": "Rename keywords",
"type": "main",
"index": 0
}
]
]
},
"Extract date": {
"main": [
[
{
"node": "Create single date items",
"type": "main",
"index": 0
}
]
]
},
"HTTP Request1": {
"main": [
[
{
"node": "Extract individual posts",
"type": "main",
"index": 0
}
]
]
},
"Rename Summary": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Rename keywords": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Merge date & links": {
"main": [
[
{
"node": "Select posts of last 7 days",
"type": "main",
"index": 0
}
]
]
},
"Create single date items": {
"main": [
[
{
"node": "Merge date & links",
"type": "main",
"index": 0
}
]
]
},
"Create single link items": {
"main": [
[
{
"node": "Merge date & links",
"type": "main",
"index": 1
}
]
]
},
"Extract individual posts": {
"main": [
[
{
"node": "Merge Content with Date & Link",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger each week": {
"main": [
[
{
"node": "Retrieve the web page for further processsing",
"type": "main",
"index": 0
}
]
]
},
"Select posts of last 7 days": {
"main": [
[
{
"node": "Merge Content with Date & Link",
"type": "main",
"index": 1
},
{
"node": "HTTP Request1",
"type": "main",
"index": 0
}
]
]
},
"Merge Content with Date & Link": {
"main": [
[
{
"node": "Summary",
"type": "main",
"index": 0
},
{
"node": "Keywords",
"type": "main",
"index": 0
},
{
"node": "Merge ChatGPT output with Date & Link",
"type": "main",
"index": 0
}
]
]
},
"Merge ChatGPT output with Date & Link": {
"main": [
[
{
"node": "NocoDB news database",
"type": "main",
"index": 0
}
]
]
},
"Extract the HTML with the right css class": {
"main": [
[
{
"node": "Create single link items",
"type": "main",
"index": 0
}
]
]
},
"Retrieve the web page for further processsing": {
"main": [
[
{
"node": "Extract the HTML with the right css class",
"type": "main",
"index": 0
},
{
"node": "Extract date",
"type": "main",
"index": 0
}
]
]
}
}
}

396
AI_Research_RAG_and_Data_Analysis/Scrape and summarize webpages with AI.json Normal file
View File

@@ -0,0 +1,396 @@
{
"meta": {
"instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9"
},
"nodes": [
{
"id": "67850bd7-f9f4-4d5b-8c9e-bd1451247ba6",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-740,
1000
],
"parameters": {},
"typeVersion": 1
},
{
"id": "0d9133f9-b6d3-4101-95c6-3cd24cdb70c3",
"name": "Fetch essay list",
"type": "n8n-nodes-base.httpRequest",
"position": [
-520,
1000
],
"parameters": {
"url": "http://www.paulgraham.com/articles.html",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "ee634297-a456-4f70-a995-55b02950571e",
"name": "Extract essay names",
"type": "n8n-nodes-base.html",
"position": [
-300,
1000
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"dataPropertyName": "=data",
"extractionValues": {
"values": [
{
"key": "essay",
"attribute": "href",
"cssSelector": "table table a",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1
},
{
"id": "83d75693-dbb8-44c4-8533-da06f611c59c",
"name": "Fetch essay texts",
"type": "n8n-nodes-base.httpRequest",
"position": [
360,
1000
],
"parameters": {
"url": "=http://www.paulgraham.com/{{ $json.essay }}",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "151022b5-8570-4176-bf3f-137f27ac7036",
"name": "Extract title",
"type": "n8n-nodes-base.html",
"position": [
700,
700
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "title",
"cssSelector": "title"
}
]
}
},
"typeVersion": 1
},
{
"id": "07bcf095-3c4d-4a72-9bcb-341411750ff5",
"name": "Clean up",
"type": "n8n-nodes-base.set",
"position": [
1360,
980
],
"parameters": {
"fields": {
"values": [
{
"name": "title",
"stringValue": "={{ $json.title }}"
},
{
"name": "summary",
"stringValue": "={{ $json.response.text }}"
},
{
"name": "url",
"stringValue": "=http://www.paulgraham.com/{{ $('Limit to first 3').item.json.essay }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3
},
{
"id": "11285de0-3c5d-4296-a322-9b7585af9acc",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-580,
920
],
"parameters": {
"width": 1071.752021563343,
"height": 285.66037735849045,
"content": "## Scrape latest Paul Graham essays"
},
"typeVersion": 1
},
{
"id": "c32f905d-dd7a-4b68-bbe0-dd8115ee0944",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
920
],
"parameters": {
"width": 465.3908355795153,
"height": 606.7924528301882,
"content": "## Summarize them with GPT"
},
"typeVersion": 1
},
{
"id": "29d264f4-df6d-4a41-ab38-58e1b1becc9a",
"name": "Split out into items",
"type": "n8n-nodes-base.splitOut",
"position": [
-80,
1000
],
"parameters": {
"options": {},
"fieldToSplitOut": "essay"
},
"typeVersion": 1
},
{
"id": "ccfa3a1d-f170-44b4-a1f2-3573c88cae98",
"name": "Limit to first 3",
"type": "n8n-nodes-base.limit",
"position": [
140,
1000
],
"parameters": {
"maxItems": 3
},
"typeVersion": 1
},
{
"id": "c3d05068-9d1a-4ef5-8249-e7384dc617ee",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
820,
1200
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "db75adad-cb16-4e72-b16e-34684a733b05",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
820,
1340
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "022cc091-9b4c-45c2-bc8e-4037ec2d0d60",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
680,
1200
],
"parameters": {
"model": "gpt-4o-mini",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "cda47bb7-36c5-4d15-a1ef-0c66b1194825",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
1160,
980
],
"parameters": {
"mode": "combine",
"options": {},
"combineBy": "combineByPosition"
},
"typeVersion": 3
},
{
"id": "28144e4c-e425-428d-b3d1-f563bfd4e5b3",
"name": "Summarization Chain",
"type": "@n8n/n8n-nodes-langchain.chainSummarization",
"position": [
720,
1000
],
"parameters": {
"options": {},
"operationMode": "documentLoader"
},
"typeVersion": 2
}
],
"pinData": {},
"connections": {
"Merge": {
"main": [
[
{
"node": "Clean up",
"type": "main",
"index": 0
}
]
]
},
"Extract title": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Fetch essay list": {
"main": [
[
{
"node": "Extract essay names",
"type": "main",
"index": 0
}
]
]
},
"Limit to first 3": {
"main": [
[
{
"node": "Fetch essay texts",
"type": "main",
"index": 0
}
]
]
},
"Fetch essay texts": {
"main": [
[
{
"node": "Extract title",
"type": "main",
"index": 0
},
{
"node": "Summarization Chain",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Summarization Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Summarization Chain",
"type": "ai_document",
"index": 0
}
]
]
},
"Extract essay names": {
"main": [
[
{
"node": "Split out into items",
"type": "main",
"index": 0
}
]
]
},
"Summarization Chain": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Split out into items": {
"main": [
[
{
"node": "Limit to first 3",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Fetch essay list",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}

396
AI_Research_RAG_and_Data_Analysis/Scrape and summarize webpages with AI.txt
View File

@@ -1,396 +0,0 @@
{
"meta": {
"instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9"
},
"nodes": [
{
"id": "67850bd7-f9f4-4d5b-8c9e-bd1451247ba6",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-740,
1000
],
"parameters": {},
"typeVersion": 1
},
{
"id": "0d9133f9-b6d3-4101-95c6-3cd24cdb70c3",
"name": "Fetch essay list",
"type": "n8n-nodes-base.httpRequest",
"position": [
-520,
1000
],
"parameters": {
"url": "http://www.paulgraham.com/articles.html",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "ee634297-a456-4f70-a995-55b02950571e",
"name": "Extract essay names",
"type": "n8n-nodes-base.html",
"position": [
-300,
1000
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"dataPropertyName": "=data",
"extractionValues": {
"values": [
{
"key": "essay",
"attribute": "href",
"cssSelector": "table table a",
"returnArray": true,
"returnValue": "attribute"
}
]
}
},
"typeVersion": 1
},
{
"id": "83d75693-dbb8-44c4-8533-da06f611c59c",
"name": "Fetch essay texts",
"type": "n8n-nodes-base.httpRequest",
"position": [
360,
1000
],
"parameters": {
"url": "=http://www.paulgraham.com/{{ $json.essay }}",
"options": {}
},
"typeVersion": 4.1
},
{
"id": "151022b5-8570-4176-bf3f-137f27ac7036",
"name": "Extract title",
"type": "n8n-nodes-base.html",
"position": [
700,
700
],
"parameters": {
"options": {},
"operation": "extractHtmlContent",
"extractionValues": {
"values": [
{
"key": "title",
"cssSelector": "title"
}
]
}
},
"typeVersion": 1
},
{
"id": "07bcf095-3c4d-4a72-9bcb-341411750ff5",
"name": "Clean up",
"type": "n8n-nodes-base.set",
"position": [
1360,
980
],
"parameters": {
"fields": {
"values": [
{
"name": "title",
"stringValue": "={{ $json.title }}"
},
{
"name": "summary",
"stringValue": "={{ $json.response.text }}"
},
{
"name": "url",
"stringValue": "=http://www.paulgraham.com/{{ $('Limit to first 3').item.json.essay }}"
}
]
},
"include": "none",
"options": {}
},
"typeVersion": 3
},
{
"id": "11285de0-3c5d-4296-a322-9b7585af9acc",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-580,
920
],
"parameters": {
"width": 1071.752021563343,
"height": 285.66037735849045,
"content": "## Scrape latest Paul Graham essays"
},
"typeVersion": 1
},
{
"id": "c32f905d-dd7a-4b68-bbe0-dd8115ee0944",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
920
],
"parameters": {
"width": 465.3908355795153,
"height": 606.7924528301882,
"content": "## Summarize them with GPT"
},
"typeVersion": 1
},
{
"id": "29d264f4-df6d-4a41-ab38-58e1b1becc9a",
"name": "Split out into items",
"type": "n8n-nodes-base.splitOut",
"position": [
-80,
1000
],
"parameters": {
"options": {},
"fieldToSplitOut": "essay"
},
"typeVersion": 1
},
{
"id": "ccfa3a1d-f170-44b4-a1f2-3573c88cae98",
"name": "Limit to first 3",
"type": "n8n-nodes-base.limit",
"position": [
140,
1000
],
"parameters": {
"maxItems": 3
},
"typeVersion": 1
},
{
"id": "c3d05068-9d1a-4ef5-8249-e7384dc617ee",
"name": "Default Data Loader",
"type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
"position": [
820,
1200
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "db75adad-cb16-4e72-b16e-34684a733b05",
"name": "Recursive Character Text Splitter",
"type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
"position": [
820,
1340
],
"parameters": {
"options": {}
},
"typeVersion": 1
},
{
"id": "022cc091-9b4c-45c2-bc8e-4037ec2d0d60",
"name": "OpenAI Chat Model1",
"type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
"position": [
680,
1200
],
"parameters": {
"model": "gpt-4o-mini",
"options": {}
},
"credentials": {
"openAiApi": {
"id": "8gccIjcuf3gvaoEr",
"name": "OpenAi account"
}
},
"typeVersion": 1
},
{
"id": "cda47bb7-36c5-4d15-a1ef-0c66b1194825",
"name": "Merge",
"type": "n8n-nodes-base.merge",
"position": [
1160,
980
],
"parameters": {
"mode": "combine",
"options": {},
"combineBy": "combineByPosition"
},
"typeVersion": 3
},
{
"id": "28144e4c-e425-428d-b3d1-f563bfd4e5b3",
"name": "Summarization Chain",
"type": "@n8n/n8n-nodes-langchain.chainSummarization",
"position": [
720,
1000
],
"parameters": {
"options": {},
"operationMode": "documentLoader"
},
"typeVersion": 2
}
],
"pinData": {},
"connections": {
"Merge": {
"main": [
[
{
"node": "Clean up",
"type": "main",
"index": 0
}
]
]
},
"Extract title": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 0
}
]
]
},
"Fetch essay list": {
"main": [
[
{
"node": "Extract essay names",
"type": "main",
"index": 0
}
]
]
},
"Limit to first 3": {
"main": [
[
{
"node": "Fetch essay texts",
"type": "main",
"index": 0
}
]
]
},
"Fetch essay texts": {
"main": [
[
{
"node": "Extract title",
"type": "main",
"index": 0
},
{
"node": "Summarization Chain",
"type": "main",
"index": 0
}
]
]
},
"OpenAI Chat Model1": {
"ai_languageModel": [
[
{
"node": "Summarization Chain",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"Default Data Loader": {
"ai_document": [
[
{
"node": "Summarization Chain",
"type": "ai_document",
"index": 0
}
]
]
},
"Extract essay names": {
"main": [
[
{
"node": "Split out into items",
"type": "main",
"index": 0
}
]
]
},
"Summarization Chain": {
"main": [
[
{
"node": "Merge",
"type": "main",
"index": 1
}
]
]
},
"Split out into items": {
"main": [
[
{
"node": "Limit to first 3",
"type": "main",
"index": 0
}
]
]
},
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Fetch essay list",
"type": "main",
"index": 0
}
]
]
},
"Recursive Character Text Splitter": {
"ai_textSplitter": [
[
{
"node": "Default Data Loader",
"type": "ai_textSplitter",
"index": 0
}
]
]
}
}
}

853
AI_Research_RAG_and_Data_Analysis/Send Google analytics data to A.I. to analyze then save results in Baserow.json Normal file
View File

@@ -0,0 +1,853 @@
{
"id": "K3uf8aY8wipScEay",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Google analytics template",
"tags": [],
"nodes": [
{
"id": "6a9fc442-d0a3-48be-8dff-94f8d9cd5cf1",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
460
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "484cbc41-f57d-4c3d-a458-e439d480d290",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
640
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b1b66e9b-5fea-407b-9c1e-39bd2a9d4a90",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
100
],
"parameters": {
"width": 714.172987012987,
"content": "## Send Google analytics to A.I. and save results to baserow\n\nThis workflow will check for country views, page engagement and google search console results. It will take this week's data and compare it to last week's data.\n\n[You can read more about this workflow here](https://rumjahn.com/how-i-used-a-i-to-be-an-seo-expert-and-analyzed-my-google-analytics-data-in-n8n-and-make-com/)"
},
"typeVersion": 1
},
{
"id": "adde29fc-ddb5-4b50-aa78-313ac9ede879",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
633.6540259740264,
320
],
"parameters": {
"color": 4,
"width": 2097.92831168831,
"height": 342.6576623376624,
"content": "## Property ID\n\n1. Create your [Google Analytics Credentials](https://docs.n8n.io/integrations/builtin/credentials/google/oauth-single-service/?utm_source=n8n_app&utm_medium=credential_settings&utm_campaign=create_new_credentials_modal)\n2. Enter your [property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id)."
},
"typeVersion": 1
},
{
"id": "f2fb8535-e81e-4ca1-80df-ee68edba6386",
"name": "Get Page Engagement Stats for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
700,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "1d761425-cebf-4787-b286-b723a0851485",
"name": "Get Page Engagement Stats for prior week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1060,
460
],
"parameters": {
"simple": false,
"endDate": "2024-10-23T00:00:00",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "f8dac36b-9e8a-407f-b923-b4cea368f1bc",
"name": "Parse data from Google Analytics",
"type": "n8n-nodes-base.code",
"position": [
880,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "ed880442-c92e-4347-b277-e8794aea6fbc",
"name": "Parse GA data",
"type": "n8n-nodes-base.code",
"position": [
1240,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "46e092cc-af94-4e64-aa92-931c56345eff",
"name": "Get Google Search Results for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1420,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "709d0aaf-bd3d-4d83-9e66-b7df495855bd",
"name": "Get Google Search Results for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1780,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7d3835d6-d1f5-4159-8e34-871871e63989",
"name": "Parse Google Analytics Data",
"type": "n8n-nodes-base.code",
"position": [
1600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "c018fda4-a2e6-48f4-aabb-039c66374dc7",
"name": "Parse Google Analytics Data1",
"type": "n8n-nodes-base.code",
"position": [
1940,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "d8f775cd-daf9-42de-a527-d932be46d945",
"name": "Get Country views data for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2120,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7119e57c-cbf4-49a9-b0c9-1f3da1fd2af3",
"name": "Get Country views data for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2440,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "546d6cd2-6db6-4276-be35-abbe5a7e9b6a",
"name": "Parse Google analytics data",
"type": "n8n-nodes-base.code",
"position": [
2280,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "87cb137c-686d-49a5-8657-06ed0c5f5c27",
"name": "Parse Google analytics data1",
"type": "n8n-nodes-base.code",
"position": [
2600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "06c4478d-a13a-4587-9f1f-451a68798a9f",
"name": "Send page data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2760,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $json.urlString }} Data from last week: {{ $('Parse data from Google Analytics').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "4ad522b0-afe4-4eff-aa16-b86cc892ead8",
"name": "Send page Search data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2920,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google Analytics Data1').item.json.urlString }} Data from last week:{{ $('Parse Google Analytics Data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "07e1eebf-f16a-44c0-83b5-76bf65a3d3fc",
"name": "Send country view data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
3080,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google analytics data1').item.json.urlString }} Data from last week:{{ $('Parse Google analytics data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c4648ad8-2377-42a0-a431-931b53631c9d",
"name": "Save A.I. output to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
3240,
460
],
"parameters": {
"tableId": 601,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5833,
"fieldValue": "Name of your blog"
},
{
"fieldId": 5831,
"fieldValue": "={{ $('Send page data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5830,
"fieldValue": "={{ $('Send page Search data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5832,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5829,
"fieldValue": "={{ DateTime.now() }}"
}
]
},
"operation": "create",
"databaseId": 121
},
"typeVersion": 1
},
{
"id": "e185c836-c12f-4452-92bd-0daaf33b653a",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2760,
180
],
"parameters": {
"color": 5,
"width": 441.7412987012988,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "a1de2d16-d09e-4c74-8be1-f6bab8c34246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
3220,
180
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Name\n- Country Views\n- Page Views\n- Search Report\n- Blog \n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ac4b5eac-1c84-49ce-9ff7-794f857265b4",
"connections": {
"Parse GA data": {
"main": [
[
{
"node": "Get Google Search Results for this week",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Send page data to A.I.": {
"main": [
[
{
"node": "Send page Search data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data": {
"main": [
[
{
"node": "Get Google Search Results for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data": {
"main": [
[
{
"node": "Get Country views data for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data1": {
"main": [
[
{
"node": "Get Country views data for this week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data1": {
"main": [
[
{
"node": "Send page data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send page Search data to A.I.": {
"main": [
[
{
"node": "Send country view data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send country view data to A.I.": {
"main": [
[
{
"node": "Save A.I. output to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Parse data from Google Analytics": {
"main": [
[
{
"node": "Get Page Engagement Stats for prior week",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for last week": {
"main": [
[
{
"node": "Parse Google analytics data1",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for this week": {
"main": [
[
{
"node": "Parse Google analytics data",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for last week": {
"main": [
[
{
"node": "Parse Google Analytics Data1",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for this week": {
"main": [
[
{
"node": "Parse Google Analytics Data",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for this week": {
"main": [
[
{
"node": "Parse data from Google Analytics",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for prior week": {
"main": [
[
{
"node": "Parse GA data",
"type": "main",
"index": 0
}
]
]
}
}
}

853
AI_Research_RAG_and_Data_Analysis/Send Google analytics data to A.I. to analyze then save results in Baserow.txt
View File

@@ -1,853 +0,0 @@
{
"id": "K3uf8aY8wipScEay",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Google analytics template",
"tags": [],
"nodes": [
{
"id": "6a9fc442-d0a3-48be-8dff-94f8d9cd5cf1",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
460
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "484cbc41-f57d-4c3d-a458-e439d480d290",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
640
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b1b66e9b-5fea-407b-9c1e-39bd2a9d4a90",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
100
],
"parameters": {
"width": 714.172987012987,
"content": "## Send Google analytics to A.I. and save results to baserow\n\nThis workflow will check for country views, page engagement and google search console results. It will take this week's data and compare it to last week's data.\n\n[You can read more about this workflow here](https://rumjahn.com/how-i-used-a-i-to-be-an-seo-expert-and-analyzed-my-google-analytics-data-in-n8n-and-make-com/)"
},
"typeVersion": 1
},
{
"id": "adde29fc-ddb5-4b50-aa78-313ac9ede879",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
633.6540259740264,
320
],
"parameters": {
"color": 4,
"width": 2097.92831168831,
"height": 342.6576623376624,
"content": "## Property ID\n\n1. Create your [Google Analytics Credentials](https://docs.n8n.io/integrations/builtin/credentials/google/oauth-single-service/?utm_source=n8n_app&utm_medium=credential_settings&utm_campaign=create_new_credentials_modal)\n2. Enter your [property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id)."
},
"typeVersion": 1
},
{
"id": "f2fb8535-e81e-4ca1-80df-ee68edba6386",
"name": "Get Page Engagement Stats for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
700,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "1d761425-cebf-4787-b286-b723a0851485",
"name": "Get Page Engagement Stats for prior week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1060,
460
],
"parameters": {
"simple": false,
"endDate": "2024-10-23T00:00:00",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "f8dac36b-9e8a-407f-b923-b4cea368f1bc",
"name": "Parse data from Google Analytics",
"type": "n8n-nodes-base.code",
"position": [
880,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "ed880442-c92e-4347-b277-e8794aea6fbc",
"name": "Parse GA data",
"type": "n8n-nodes-base.code",
"position": [
1240,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "46e092cc-af94-4e64-aa92-931c56345eff",
"name": "Get Google Search Results for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1420,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "709d0aaf-bd3d-4d83-9e66-b7df495855bd",
"name": "Get Google Search Results for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1780,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7d3835d6-d1f5-4159-8e34-871871e63989",
"name": "Parse Google Analytics Data",
"type": "n8n-nodes-base.code",
"position": [
1600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "c018fda4-a2e6-48f4-aabb-039c66374dc7",
"name": "Parse Google Analytics Data1",
"type": "n8n-nodes-base.code",
"position": [
1940,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "d8f775cd-daf9-42de-a527-d932be46d945",
"name": "Get Country views data for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2120,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7119e57c-cbf4-49a9-b0c9-1f3da1fd2af3",
"name": "Get Country views data for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2440,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "546d6cd2-6db6-4276-be35-abbe5a7e9b6a",
"name": "Parse Google analytics data",
"type": "n8n-nodes-base.code",
"position": [
2280,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "87cb137c-686d-49a5-8657-06ed0c5f5c27",
"name": "Parse Google analytics data1",
"type": "n8n-nodes-base.code",
"position": [
2600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "06c4478d-a13a-4587-9f1f-451a68798a9f",
"name": "Send page data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2760,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $json.urlString }} Data from last week: {{ $('Parse data from Google Analytics').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "4ad522b0-afe4-4eff-aa16-b86cc892ead8",
"name": "Send page Search data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2920,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google Analytics Data1').item.json.urlString }} Data from last week:{{ $('Parse Google Analytics Data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "07e1eebf-f16a-44c0-83b5-76bf65a3d3fc",
"name": "Send country view data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
3080,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google analytics data1').item.json.urlString }} Data from last week:{{ $('Parse Google analytics data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c4648ad8-2377-42a0-a431-931b53631c9d",
"name": "Save A.I. output to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
3240,
460
],
"parameters": {
"tableId": 601,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5833,
"fieldValue": "Name of your blog"
},
{
"fieldId": 5831,
"fieldValue": "={{ $('Send page data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5830,
"fieldValue": "={{ $('Send page Search data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5832,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5829,
"fieldValue": "={{ DateTime.now() }}"
}
]
},
"operation": "create",
"databaseId": 121
},
"typeVersion": 1
},
{
"id": "e185c836-c12f-4452-92bd-0daaf33b653a",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2760,
180
],
"parameters": {
"color": 5,
"width": 441.7412987012988,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "a1de2d16-d09e-4c74-8be1-f6bab8c34246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
3220,
180
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Name\n- Country Views\n- Page Views\n- Search Report\n- Blog \n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ac4b5eac-1c84-49ce-9ff7-794f857265b4",
"connections": {
"Parse GA data": {
"main": [
[
{
"node": "Get Google Search Results for this week",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Send page data to A.I.": {
"main": [
[
{
"node": "Send page Search data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data": {
"main": [
[
{
"node": "Get Google Search Results for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data": {
"main": [
[
{
"node": "Get Country views data for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data1": {
"main": [
[
{
"node": "Get Country views data for this week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data1": {
"main": [
[
{
"node": "Send page data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send page Search data to A.I.": {
"main": [
[
{
"node": "Send country view data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send country view data to A.I.": {
"main": [
[
{
"node": "Save A.I. output to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Parse data from Google Analytics": {
"main": [
[
{
"node": "Get Page Engagement Stats for prior week",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for last week": {
"main": [
[
{
"node": "Parse Google analytics data1",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for this week": {
"main": [
[
{
"node": "Parse Google analytics data",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for last week": {
"main": [
[
{
"node": "Parse Google Analytics Data1",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for this week": {
"main": [
[
{
"node": "Parse Google Analytics Data",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for this week": {
"main": [
[
{
"node": "Parse data from Google Analytics",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for prior week": {
"main": [
[
{
"node": "Parse GA data",
"type": "main",
"index": 0
}
]
]
}
}
}

853
AI_Research_RAG_and_Data_Analysis/Send Google analytics data to A.I. to analyze then save results in BaserowSend Google analytics data to A.I. to analyze then save results in Baserow.json Normal file
View File

@@ -0,0 +1,853 @@
{
"id": "K3uf8aY8wipScEay",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Google analytics template",
"tags": [],
"nodes": [
{
"id": "6a9fc442-d0a3-48be-8dff-94f8d9cd5cf1",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
460
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "484cbc41-f57d-4c3d-a458-e439d480d290",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
640
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b1b66e9b-5fea-407b-9c1e-39bd2a9d4a90",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
100
],
"parameters": {
"width": 714.172987012987,
"content": "## Send Google analytics to A.I. and save results to baserow\n\nThis workflow will check for country views, page engagement and google search console results. It will take this week's data and compare it to last week's data.\n\n[You can read more about this workflow here](https://rumjahn.com/how-i-used-a-i-to-be-an-seo-expert-and-analyzed-my-google-analytics-data-in-n8n-and-make-com/)"
},
"typeVersion": 1
},
{
"id": "adde29fc-ddb5-4b50-aa78-313ac9ede879",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
633.6540259740264,
320
],
"parameters": {
"color": 4,
"width": 2097.92831168831,
"height": 342.6576623376624,
"content": "## Property ID\n\n1. Create your [Google Analytics Credentials](https://docs.n8n.io/integrations/builtin/credentials/google/oauth-single-service/?utm_source=n8n_app&utm_medium=credential_settings&utm_campaign=create_new_credentials_modal)\n2. Enter your [property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id)."
},
"typeVersion": 1
},
{
"id": "f2fb8535-e81e-4ca1-80df-ee68edba6386",
"name": "Get Page Engagement Stats for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
700,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "1d761425-cebf-4787-b286-b723a0851485",
"name": "Get Page Engagement Stats for prior week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1060,
460
],
"parameters": {
"simple": false,
"endDate": "2024-10-23T00:00:00",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "f8dac36b-9e8a-407f-b923-b4cea368f1bc",
"name": "Parse data from Google Analytics",
"type": "n8n-nodes-base.code",
"position": [
880,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "ed880442-c92e-4347-b277-e8794aea6fbc",
"name": "Parse GA data",
"type": "n8n-nodes-base.code",
"position": [
1240,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "46e092cc-af94-4e64-aa92-931c56345eff",
"name": "Get Google Search Results for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1420,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "709d0aaf-bd3d-4d83-9e66-b7df495855bd",
"name": "Get Google Search Results for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1780,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7d3835d6-d1f5-4159-8e34-871871e63989",
"name": "Parse Google Analytics Data",
"type": "n8n-nodes-base.code",
"position": [
1600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "c018fda4-a2e6-48f4-aabb-039c66374dc7",
"name": "Parse Google Analytics Data1",
"type": "n8n-nodes-base.code",
"position": [
1940,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "d8f775cd-daf9-42de-a527-d932be46d945",
"name": "Get Country views data for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2120,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7119e57c-cbf4-49a9-b0c9-1f3da1fd2af3",
"name": "Get Country views data for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2440,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "546d6cd2-6db6-4276-be35-abbe5a7e9b6a",
"name": "Parse Google analytics data",
"type": "n8n-nodes-base.code",
"position": [
2280,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "87cb137c-686d-49a5-8657-06ed0c5f5c27",
"name": "Parse Google analytics data1",
"type": "n8n-nodes-base.code",
"position": [
2600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "06c4478d-a13a-4587-9f1f-451a68798a9f",
"name": "Send page data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2760,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $json.urlString }} Data from last week: {{ $('Parse data from Google Analytics').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "4ad522b0-afe4-4eff-aa16-b86cc892ead8",
"name": "Send page Search data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2920,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google Analytics Data1').item.json.urlString }} Data from last week:{{ $('Parse Google Analytics Data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "07e1eebf-f16a-44c0-83b5-76bf65a3d3fc",
"name": "Send country view data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
3080,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google analytics data1').item.json.urlString }} Data from last week:{{ $('Parse Google analytics data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c4648ad8-2377-42a0-a431-931b53631c9d",
"name": "Save A.I. output to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
3240,
460
],
"parameters": {
"tableId": 601,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5833,
"fieldValue": "Name of your blog"
},
{
"fieldId": 5831,
"fieldValue": "={{ $('Send page data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5830,
"fieldValue": "={{ $('Send page Search data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5832,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5829,
"fieldValue": "={{ DateTime.now() }}"
}
]
},
"operation": "create",
"databaseId": 121
},
"typeVersion": 1
},
{
"id": "e185c836-c12f-4452-92bd-0daaf33b653a",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2760,
180
],
"parameters": {
"color": 5,
"width": 441.7412987012988,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "a1de2d16-d09e-4c74-8be1-f6bab8c34246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
3220,
180
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Name\n- Country Views\n- Page Views\n- Search Report\n- Blog \n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ac4b5eac-1c84-49ce-9ff7-794f857265b4",
"connections": {
"Parse GA data": {
"main": [
[
{
"node": "Get Google Search Results for this week",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Send page data to A.I.": {
"main": [
[
{
"node": "Send page Search data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data": {
"main": [
[
{
"node": "Get Google Search Results for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data": {
"main": [
[
{
"node": "Get Country views data for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data1": {
"main": [
[
{
"node": "Get Country views data for this week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data1": {
"main": [
[
{
"node": "Send page data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send page Search data to A.I.": {
"main": [
[
{
"node": "Send country view data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send country view data to A.I.": {
"main": [
[
{
"node": "Save A.I. output to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Parse data from Google Analytics": {
"main": [
[
{
"node": "Get Page Engagement Stats for prior week",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for last week": {
"main": [
[
{
"node": "Parse Google analytics data1",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for this week": {
"main": [
[
{
"node": "Parse Google analytics data",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for last week": {
"main": [
[
{
"node": "Parse Google Analytics Data1",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for this week": {
"main": [
[
{
"node": "Parse Google Analytics Data",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for this week": {
"main": [
[
{
"node": "Parse data from Google Analytics",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for prior week": {
"main": [
[
{
"node": "Parse GA data",
"type": "main",
"index": 0
}
]
]
}
}
}

853
AI_Research_RAG_and_Data_Analysis/Send Google analytics data to A.I. to analyze then save results in BaserowSend Google analytics data to A.I. to analyze then save results in Baserow.txt
View File

@@ -1,853 +0,0 @@
{
"id": "K3uf8aY8wipScEay",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Google analytics template",
"tags": [],
"nodes": [
{
"id": "6a9fc442-d0a3-48be-8dff-94f8d9cd5cf1",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
460
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "484cbc41-f57d-4c3d-a458-e439d480d290",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
640
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b1b66e9b-5fea-407b-9c1e-39bd2a9d4a90",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
460,
100
],
"parameters": {
"width": 714.172987012987,
"content": "## Send Google analytics to A.I. and save results to baserow\n\nThis workflow will check for country views, page engagement and google search console results. It will take this week's data and compare it to last week's data.\n\n[You can read more about this workflow here](https://rumjahn.com/how-i-used-a-i-to-be-an-seo-expert-and-analyzed-my-google-analytics-data-in-n8n-and-make-com/)"
},
"typeVersion": 1
},
{
"id": "adde29fc-ddb5-4b50-aa78-313ac9ede879",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
633.6540259740264,
320
],
"parameters": {
"color": 4,
"width": 2097.92831168831,
"height": 342.6576623376624,
"content": "## Property ID\n\n1. Create your [Google Analytics Credentials](https://docs.n8n.io/integrations/builtin/credentials/google/oauth-single-service/?utm_source=n8n_app&utm_medium=credential_settings&utm_campaign=create_new_credentials_modal)\n2. Enter your [property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id)."
},
"typeVersion": 1
},
{
"id": "f2fb8535-e81e-4ca1-80df-ee68edba6386",
"name": "Get Page Engagement Stats for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
700,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "1d761425-cebf-4787-b286-b723a0851485",
"name": "Get Page Engagement Stats for prior week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1060,
460
],
"parameters": {
"simple": false,
"endDate": "2024-10-23T00:00:00",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "screenPageViews",
"listName": "other"
},
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "screenPageViewsPerUser",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "unifiedScreenName",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "f8dac36b-9e8a-407f-b923-b4cea368f1bc",
"name": "Parse data from Google Analytics",
"type": "n8n-nodes-base.code",
"position": [
880,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "ed880442-c92e-4347-b277-e8794aea6fbc",
"name": "Parse GA data",
"type": "n8n-nodes-base.code",
"position": [
1240,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // Debug logging\n console.log('Input items:', JSON.stringify(items, null, 2));\n \n // Check if items is an array and has content\n if (!Array.isArray(items) || items.length === 0) {\n console.log('Items is not an array or is empty');\n throw new Error('Invalid data structure');\n }\n\n // Check if first item exists and has json property\n if (!items[0] || !items[0].json) {\n console.log('First item is missing or has no json property');\n throw new Error('Invalid data structure');\n }\n\n // Get the analytics data\n const analyticsData = items[0].json;\n \n // Check if analyticsData has rows\n if (!analyticsData || !Array.isArray(analyticsData.rows)) {\n console.log('Analytics data is missing or has no rows array');\n throw new Error('Invalid data structure');\n }\n \n // Map each row to a simplified object\n const simplified = analyticsData.rows.map(row => {\n if (!row.dimensionValues?.[0]?.value || !row.metricValues?.length) {\n console.log('Invalid row structure:', row);\n throw new Error('Invalid row structure');\n }\n \n return {\n page: row.dimensionValues[0].value,\n pageViews: parseInt(row.metricValues[0].value) || 0,\n activeUsers: parseInt(row.metricValues[1].value) || 0,\n viewsPerUser: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0\n };\n });\n \n // Convert to JSON string and encode for URL\n return encodeURIComponent(JSON.stringify(simplified));\n}\n\n// Get input data and transform it\nconst urlString = transformToUrlString($input.all());\n\n// Return the result\nreturn { json: { urlString } };"
},
"typeVersion": 2
},
{
"id": "46e092cc-af94-4e64-aa92-931c56345eff",
"name": "Get Google Search Results for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1420,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "709d0aaf-bd3d-4d83-9e66-b7df495855bd",
"name": "Get Google Search Results for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
1780,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"name": "organicGoogleSearchAveragePosition",
"listName": "other"
},
{
"name": "organicGoogleSearchClickThroughRate",
"listName": "other"
},
{
"name": "organicGoogleSearchClicks",
"listName": "other"
},
{
"name": "organicGoogleSearchImpressions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "landingPagePlusQueryString",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7d3835d6-d1f5-4159-8e34-871871e63989",
"name": "Parse Google Analytics Data",
"type": "n8n-nodes-base.code",
"position": [
1600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "c018fda4-a2e6-48f4-aabb-039c66374dc7",
"name": "Parse Google Analytics Data1",
"type": "n8n-nodes-base.code",
"position": [
1940,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n page: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n engagedSessions: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n eventCount: parseInt(row.metricValues[3].value) || 0,\n avgPosition: parseFloat(row.metricValues[4].value) || 0,\n ctr: parseFloat(row.metricValues[5].value) || 0,\n clicks: parseInt(row.metricValues[6].value) || 0,\n impressions: parseInt(row.metricValues[7].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "d8f775cd-daf9-42de-a527-d932be46d945",
"name": "Get Country views data for this week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2120,
460
],
"parameters": {
"simple": false,
"returnAll": true,
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"credentials": {
"googleAnalyticsOAuth2": {
"id": "b1GX8VBMKCUNweV1",
"name": "Google Analytics account"
}
},
"typeVersion": 2
},
{
"id": "7119e57c-cbf4-49a9-b0c9-1f3da1fd2af3",
"name": "Get Country views data for last week",
"type": "n8n-nodes-base.googleAnalytics",
"position": [
2440,
460
],
"parameters": {
"simple": false,
"endDate": "={{$today.minus({days: 7})}}",
"dateRange": "custom",
"returnAll": true,
"startDate": "={{$today.minus({days: 14})}}",
"metricsGA4": {
"metricValues": [
{
"name": "activeUsers",
"listName": "other"
},
{
"name": "newUsers",
"listName": "other"
},
{
"name": "engagementRate",
"listName": "other"
},
{
"name": "engagedSessions",
"listName": "other"
},
{
"name": "eventCount",
"listName": "other"
},
{
"listName": "other"
},
{
"name": "sessions",
"listName": "other"
}
]
},
"propertyId": {
"__rl": true,
"mode": "id",
"value": "460520224"
},
"dimensionsGA4": {
"dimensionValues": [
{
"name": "country",
"listName": "other"
}
]
},
"additionalFields": {}
},
"typeVersion": 2
},
{
"id": "546d6cd2-6db6-4276-be35-abbe5a7e9b6a",
"name": "Parse Google analytics data",
"type": "n8n-nodes-base.code",
"position": [
2280,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "87cb137c-686d-49a5-8657-06ed0c5f5c27",
"name": "Parse Google analytics data1",
"type": "n8n-nodes-base.code",
"position": [
2600,
460
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data || !data.rows) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Process each row\n const simplified = data.rows.map(row => ({\n country: row.dimensionValues[0].value,\n activeUsers: parseInt(row.metricValues[0].value) || 0,\n newUsers: parseInt(row.metricValues[1].value) || 0,\n engagementRate: parseFloat(row.metricValues[2].value) || 0,\n engagedSessions: parseInt(row.metricValues[3].value) || 0,\n eventCount: parseInt(row.metricValues[4].value) || 0,\n totalUsers: parseInt(row.metricValues[5].value) || 0,\n sessions: parseInt(row.metricValues[6].value) || 0\n }));\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "06c4478d-a13a-4587-9f1f-451a68798a9f",
"name": "Send page data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2760,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $json.urlString }} Data from last week: {{ $('Parse data from Google Analytics').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "4ad522b0-afe4-4eff-aa16-b86cc892ead8",
"name": "Send page Search data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
2920,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google Analytics Data1').item.json.urlString }} Data from last week:{{ $('Parse Google Analytics Data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "07e1eebf-f16a-44c0-83b5-76bf65a3d3fc",
"name": "Send country view data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
3080,
460
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Compare the data from past 2 weeks, give me a table in markdown. Then give me 5 suggestions to improve my SEO. Output the data so that it works with markdown editors. Data from 2 weeks ago:{{ $('Parse Google analytics data1').item.json.urlString }} Data from last week:{{ $('Parse Google analytics data').item.json.urlString }}\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c4648ad8-2377-42a0-a431-931b53631c9d",
"name": "Save A.I. output to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
3240,
460
],
"parameters": {
"tableId": 601,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5833,
"fieldValue": "Name of your blog"
},
{
"fieldId": 5831,
"fieldValue": "={{ $('Send page data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5830,
"fieldValue": "={{ $('Send page Search data to A.I.').item.json.choices[0].message.content }}"
},
{
"fieldId": 5832,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5829,
"fieldValue": "={{ DateTime.now() }}"
}
]
},
"operation": "create",
"databaseId": 121
},
"typeVersion": 1
},
{
"id": "e185c836-c12f-4452-92bd-0daaf33b653a",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
2760,
180
],
"parameters": {
"color": 5,
"width": 441.7412987012988,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "a1de2d16-d09e-4c74-8be1-f6bab8c34246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
3220,
180
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Name\n- Country Views\n- Page Views\n- Search Report\n- Blog \n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "ac4b5eac-1c84-49ce-9ff7-794f857265b4",
"connections": {
"Parse GA data": {
"main": [
[
{
"node": "Get Google Search Results for this week",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Send page data to A.I.": {
"main": [
[
{
"node": "Send page Search data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data": {
"main": [
[
{
"node": "Get Google Search Results for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data": {
"main": [
[
{
"node": "Get Country views data for last week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google Analytics Data1": {
"main": [
[
{
"node": "Get Country views data for this week",
"type": "main",
"index": 0
}
]
]
},
"Parse Google analytics data1": {
"main": [
[
{
"node": "Send page data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send page Search data to A.I.": {
"main": [
[
{
"node": "Send country view data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Send country view data to A.I.": {
"main": [
[
{
"node": "Save A.I. output to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Parse data from Google Analytics": {
"main": [
[
{
"node": "Get Page Engagement Stats for prior week",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get Page Engagement Stats for this week",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for last week": {
"main": [
[
{
"node": "Parse Google analytics data1",
"type": "main",
"index": 0
}
]
]
},
"Get Country views data for this week": {
"main": [
[
{
"node": "Parse Google analytics data",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for last week": {
"main": [
[
{
"node": "Parse Google Analytics Data1",
"type": "main",
"index": 0
}
]
]
},
"Get Google Search Results for this week": {
"main": [
[
{
"node": "Parse Google Analytics Data",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for this week": {
"main": [
[
{
"node": "Parse data from Google Analytics",
"type": "main",
"index": 0
}
]
]
},
"Get Page Engagement Stats for prior week": {
"main": [
[
{
"node": "Parse GA data",
"type": "main",
"index": 0
}
]
]
}
}
}

1850
AI_Research_RAG_and_Data_Analysis/Spot Workplace Discrimination Patterns with AI.json Normal file
View File

File diff suppressed because one or more lines are too long

1850
AI_Research_RAG_and_Data_Analysis/Spot Workplace Discrimination Patterns with AI.txt
View File

File diff suppressed because one or more lines are too long

277
AI_Research_RAG_and_Data_Analysis/Summarize SERPBear data with AI (via Openrouter) and save it to Baserow.json Normal file
View File

@@ -0,0 +1,277 @@
{
"id": "qmmXKcpJOCm9qaCk",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "SERPBear analytics template",
"tags": [],
"nodes": [
{
"id": "2ad0eb40-6628-4c6b-bc15-7081e7712f1a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
260,
380
],
"parameters": {},
"typeVersion": 1
},
{
"id": "5a3c9ad8-a562-4bb0-bb11-c325552d8101",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
260,
160
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "bdfa7388-f9b3-4145-90de-2e58138e14bf",
"name": "Get data from SerpBear",
"type": "n8n-nodes-base.httpRequest",
"position": [
580,
260
],
"parameters": {
"url": "https://myserpbearinstance.com/api/keyword?id=22",
"options": {},
"sendQuery": true,
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"queryParameters": {
"parameters": [
{
"name": "domain",
"value": "rumjahn.com"
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "3fshHb4fyI5XfLyq",
"name": "Header Auth account 6"
}
},
"executeOnce": false,
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c169f4e3-ab60-4b46-9f49-cf27a13dd7c6",
"name": "Parse data from SerpBear",
"type": "n8n-nodes-base.code",
"position": [
820,
260
],
"parameters": {
"jsCode": "const keywords = items[0].json.keywords;\nconst today = new Date().toISOString().split('T')[0];\n\n// Create summary for each keyword\nconst keywordSummaries = keywords.map(kw => {\n const position = kw.position || 0;\n const lastWeekPositions = Object.values(kw.history || {}).slice(-7);\n const avgPosition = lastWeekPositions.reduce((a, b) => a + b, 0) / lastWeekPositions.length;\n \n return {\n keyword: kw.keyword,\n currentPosition: position,\n averagePosition: Math.round(avgPosition * 10) / 10,\n trend: position < avgPosition ? 'improving' : position > avgPosition ? 'declining' : 'stable',\n url: kw.url || 'not ranking'\n };\n});\n\n// Create the prompt\nconst prompt = `Here's the SEO ranking data for rumjahn.com as of ${today}:\n\n${keywordSummaries.map(kw => `\nKeyword: \"${kw.keyword}\"\nCurrent Position: ${kw.currentPosition}\n7-Day Average: ${kw.averagePosition}\nTrend: ${kw.trend}\nRanking URL: ${kw.url}\n`).join('\\n')}\n\nPlease analyze this data and provide:\n1. Key observations about ranking performance\n2. Keywords showing the most improvement\n3. Keywords needing attention\n4. Suggested actions for improvement`;\n\nreturn {\n prompt\n};"
},
"typeVersion": 2
},
{
"id": "cc6e16a7-db46-42fe-837a-59ce635c906c",
"name": "Send data to A.I. for analysis",
"type": "n8n-nodes-base.httpRequest",
"position": [
1060,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. This is keyword data for my site. Can you summarize the data into a table and then give me some suggestions:{{ encodeURIComponent($json.prompt)}}\" \n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "a623f06c-1dfe-4d04-a7fd-fed7049a7588",
"name": "Save data to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
1340,
260
],
"parameters": {
"tableId": 644,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 6264,
"fieldValue": "={{ DateTime.now().toFormat('yyyy-MM-dd') }}"
},
{
"fieldId": 6265,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 6266,
"fieldValue": "Rumjahn"
}
]
},
"operation": "create",
"databaseId": 121
},
"credentials": {
"baserowApi": {
"id": "8w0zXhycIfCAgja3",
"name": "Baserow account"
}
},
"typeVersion": 1
},
{
"id": "e8048faf-bbed-4e48-b273-d1a50a767e76",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
-360
],
"parameters": {
"color": 5,
"width": 614.709677419355,
"height": 208.51612903225802,
"content": "## Send Matomo analytics to A.I. and save results to baserow\n\nThis workflow will check the Google keywords for your site and it's rank.\n\n[💡 You can read more about this workflow here](https://rumjahn.com/how-to-create-an-a-i-agent-to-analyze-serpbear-keyword-rankings-using-n8n-for-free-without-any-coding-skills-required/)"
},
"typeVersion": 1
},
{
"id": "1a18e685-79db-423f-992a-5e0d4ddeb672",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
520,
-80
],
"parameters": {
"width": 214.75050403225822,
"height": 531.7318548387107,
"content": "## Get SERPBear Data\n \n1. Enter your SerpBear API keys and URL. You need to find your website ID which is probably 1.\n2. Navigate to Administration > Personal > Security > Auth tokens within your Matomo dashboard. Click on Create new token and provide a purpose for reference."
},
"typeVersion": 1
},
{
"id": "99895baf-75d0-4af2-87de-5b8951186e78",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
980,
-60
],
"parameters": {
"color": 3,
"width": 225.99936321742769,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "07d03511-98b0-4f4a-8e68-96ca177fb246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
-40
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Date\n- Note\n- Blog\n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "8b7e7da7-1965-4ca4-8e15-889eda819723",
"connections": {
"Schedule Trigger": {
"main": [
[
{
"node": "Get data from SerpBear",
"type": "main",
"index": 0
}
]
]
},
"Get data from SerpBear": {
"main": [
[
{
"node": "Parse data from SerpBear",
"type": "main",
"index": 0
}
]
]
},
"Parse data from SerpBear": {
"main": [
[
{
"node": "Send data to A.I. for analysis",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I. for analysis": {
"main": [
[
{
"node": "Save data to Baserow",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get data from SerpBear",
"type": "main",
"index": 0
}
]
]
}
}
}

277
AI_Research_RAG_and_Data_Analysis/Summarize SERPBear data with AI (via Openrouter) and save it to Baserow.txt
View File

@@ -1,277 +0,0 @@
{
"id": "qmmXKcpJOCm9qaCk",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "SERPBear analytics template",
"tags": [],
"nodes": [
{
"id": "2ad0eb40-6628-4c6b-bc15-7081e7712f1a",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
260,
380
],
"parameters": {},
"typeVersion": 1
},
{
"id": "5a3c9ad8-a562-4bb0-bb11-c325552d8101",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
260,
160
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks"
}
]
}
},
"typeVersion": 1.2
},
{
"id": "bdfa7388-f9b3-4145-90de-2e58138e14bf",
"name": "Get data from SerpBear",
"type": "n8n-nodes-base.httpRequest",
"position": [
580,
260
],
"parameters": {
"url": "https://myserpbearinstance.com/api/keyword?id=22",
"options": {},
"sendQuery": true,
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"queryParameters": {
"parameters": [
{
"name": "domain",
"value": "rumjahn.com"
}
]
}
},
"credentials": {
"httpHeaderAuth": {
"id": "3fshHb4fyI5XfLyq",
"name": "Header Auth account 6"
}
},
"executeOnce": false,
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "c169f4e3-ab60-4b46-9f49-cf27a13dd7c6",
"name": "Parse data from SerpBear",
"type": "n8n-nodes-base.code",
"position": [
820,
260
],
"parameters": {
"jsCode": "const keywords = items[0].json.keywords;\nconst today = new Date().toISOString().split('T')[0];\n\n// Create summary for each keyword\nconst keywordSummaries = keywords.map(kw => {\n const position = kw.position || 0;\n const lastWeekPositions = Object.values(kw.history || {}).slice(-7);\n const avgPosition = lastWeekPositions.reduce((a, b) => a + b, 0) / lastWeekPositions.length;\n \n return {\n keyword: kw.keyword,\n currentPosition: position,\n averagePosition: Math.round(avgPosition * 10) / 10,\n trend: position < avgPosition ? 'improving' : position > avgPosition ? 'declining' : 'stable',\n url: kw.url || 'not ranking'\n };\n});\n\n// Create the prompt\nconst prompt = `Here's the SEO ranking data for rumjahn.com as of ${today}:\n\n${keywordSummaries.map(kw => `\nKeyword: \"${kw.keyword}\"\nCurrent Position: ${kw.currentPosition}\n7-Day Average: ${kw.averagePosition}\nTrend: ${kw.trend}\nRanking URL: ${kw.url}\n`).join('\\n')}\n\nPlease analyze this data and provide:\n1. Key observations about ranking performance\n2. Keywords showing the most improvement\n3. Keywords needing attention\n4. Suggested actions for improvement`;\n\nreturn {\n prompt\n};"
},
"typeVersion": 2
},
{
"id": "cc6e16a7-db46-42fe-837a-59ce635c906c",
"name": "Send data to A.I. for analysis",
"type": "n8n-nodes-base.httpRequest",
"position": [
1060,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. This is keyword data for my site. Can you summarize the data into a table and then give me some suggestions:{{ encodeURIComponent($json.prompt)}}\" \n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2,
"alwaysOutputData": false
},
{
"id": "a623f06c-1dfe-4d04-a7fd-fed7049a7588",
"name": "Save data to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
1340,
260
],
"parameters": {
"tableId": 644,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 6264,
"fieldValue": "={{ DateTime.now().toFormat('yyyy-MM-dd') }}"
},
{
"fieldId": 6265,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 6266,
"fieldValue": "Rumjahn"
}
]
},
"operation": "create",
"databaseId": 121
},
"credentials": {
"baserowApi": {
"id": "8w0zXhycIfCAgja3",
"name": "Baserow account"
}
},
"typeVersion": 1
},
{
"id": "e8048faf-bbed-4e48-b273-d1a50a767e76",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
220,
-360
],
"parameters": {
"color": 5,
"width": 614.709677419355,
"height": 208.51612903225802,
"content": "## Send Matomo analytics to A.I. and save results to baserow\n\nThis workflow will check the Google keywords for your site and it's rank.\n\n[💡 You can read more about this workflow here](https://rumjahn.com/how-to-create-an-a-i-agent-to-analyze-serpbear-keyword-rankings-using-n8n-for-free-without-any-coding-skills-required/)"
},
"typeVersion": 1
},
{
"id": "1a18e685-79db-423f-992a-5e0d4ddeb672",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
520,
-80
],
"parameters": {
"width": 214.75050403225822,
"height": 531.7318548387107,
"content": "## Get SERPBear Data\n \n1. Enter your SerpBear API keys and URL. You need to find your website ID which is probably 1.\n2. Navigate to Administration > Personal > Security > Auth tokens within your Matomo dashboard. Click on Create new token and provide a purpose for reference."
},
"typeVersion": 1
},
{
"id": "99895baf-75d0-4af2-87de-5b8951186e78",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
980,
-60
],
"parameters": {
"color": 3,
"width": 225.99936321742769,
"height": 508.95792207792226,
"content": "## Send data to A.I.\n\nFill in your Openrouter A.I. credentials. Use Header Auth.\n- Username: Authorization\n- Password: Bearer {insert your API key}\n\nRemember to add a space after bearer. Also, feel free to modify the prompt to A.1."
},
"typeVersion": 1
},
{
"id": "07d03511-98b0-4f4a-8e68-96ca177fb246",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
-40
],
"parameters": {
"color": 6,
"width": 331.32883116883124,
"height": 474.88,
"content": "## Send data to Baserow\n\nCreate a table first with the following columns:\n- Date\n- Note\n- Blog\n\nEnter the name of your website under \"Blog\" field."
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "8b7e7da7-1965-4ca4-8e15-889eda819723",
"connections": {
"Schedule Trigger": {
"main": [
[
{
"node": "Get data from SerpBear",
"type": "main",
"index": 0
}
]
]
},
"Get data from SerpBear": {
"main": [
[
{
"node": "Parse data from SerpBear",
"type": "main",
"index": 0
}
]
]
},
"Parse data from SerpBear": {
"main": [
[
{
"node": "Send data to A.I. for analysis",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I. for analysis": {
"main": [
[
{
"node": "Save data to Baserow",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get data from SerpBear",
"type": "main",
"index": 0
}
]
]
}
}
}

454
AI_Research_RAG_and_Data_Analysis/Summarize Umami data with AI (via Openrouter) and save it to Baserow.json Normal file
View File

@@ -0,0 +1,454 @@
{
"id": "eZT6SZ4Kvmq5TzyQ",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Umami analytics template",
"tags": [],
"nodes": [
{
"id": "8a54ac1c-a072-42e6-a3ba-8cde33475eb5",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
220
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e81c9be0-f59d-467e-9bda-eeb2d66ed31e",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
380
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
4
]
}
]
}
},
"typeVersion": 1.2
},
{
"id": "01b04872-9aea-4834-8df5-f6c91914133d",
"name": "Get view stats from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
760,
260
],
"parameters": {
"url": "=https://umami.mydomain.com/api/websites/86d4095c-a2a8-4fc8-9521-103e858e2b41/event-data/stats?startAt={{ DateTime.now().minus({ days: 7 }).toMillis() }}&endAt={{ DateTime.now().toMillis() }}&unit=hour&timezone=Asia%2FHong_Kong",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4.2
},
{
"id": "38d342e3-10ad-4260-8f44-5a3233ec3166",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
660,
-260
],
"parameters": {
"width": 504.88636363636317,
"content": "## Send data from Umami to A.I. and then save to Baserow\n\nYou can find out more about the stats available in the [Umami API](https://umami.is/docs/api/website-stats-api)\n\nRead the [case study here](https://rumjahn.com/how-to-analyze-umami-data-using-n8n-and-a-i-to-improve-seo-and-uncover-hidden-insights-for-better-content-optimization/).\n\n"
},
"typeVersion": 1
},
{
"id": "18c997fe-61b1-464a-8bb5-fcdc017dd1f6",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
660,
-60
],
"parameters": {
"color": 4,
"width": 393.16558441558414,
"height": 504.17207792207796,
"content": "## Get summary stats from Umami\n\nIt will get: Pageviews, Visitors, Visits, Bounces, Total Time\n\nYou need to change the URL to your website. https://{your website}/api/websites/{website ID}/\n\nYou can find your ID by going to your Umami account -> Settings -> Edit (next to domain)"
},
"typeVersion": 1
},
{
"id": "bfdc04a2-57fa-4a8a-b412-39047cebb370",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1080,
-60
],
"parameters": {
"color": 5,
"width": 216.5746753246753,
"height": 502.37012987012963,
"content": "## Send data to A.I.\n\nTo use Openrouter, you need to register for an account.\nThen add header authorization credentials.\nUsername: Authroization\nPassword: Bearer {Your API Key}\n*It's Bearer space {API key}."
},
"typeVersion": 1
},
{
"id": "fc373fd7-52fc-4729-8022-021c09d0c89c",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1320,
-60
],
"parameters": {
"color": 6,
"width": 746.3474025974022,
"height": 505.9740259740257,
"content": "## Get page specific stats for this week and last\n\nCalls Umami to get this week and last week's data. It will get the views for each page visited on your website for comparison."
},
"typeVersion": 1
},
{
"id": "82bd35b6-8b49-4d77-8be2-033a8bff3f41",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
-60
],
"parameters": {
"color": 5,
"width": 216.5746753246753,
"height": 502.37012987012963,
"content": "## Send data to A.I.\n\nTo use Openrouter, you need to register for an account.\nThen add header authorization credentials.\nUsername: Authroization\nPassword: Bearer {Your API Key}\n*It's Bearer space {API key}."
},
"typeVersion": 1
},
{
"id": "503c4ca3-36da-41a8-9029-f844a34daa59",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2380,
-60
],
"parameters": {
"color": 4,
"width": 393.16558441558414,
"height": 504.17207792207796,
"content": "## Save analysis to baserow\n\nYou need to create a table in advance to save. \n- Date (date)\n- Summary (Long text)\n- Top pages (Long text)\n- Blog name (Long text)"
},
"typeVersion": 1
},
{
"id": "f64cdfbd-712f-461c-b025-25f37e2bded8",
"name": "Parse Umami data",
"type": "n8n-nodes-base.code",
"position": [
940,
260
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Create a simplified object with the metrics\n const simplified = {\n pageviews: {\n value: parseInt(data.pageviews.value) || 0,\n prev: parseInt(data.pageviews.prev) || 0\n },\n visitors: {\n value: parseInt(data.visitors.value) || 0,\n prev: parseInt(data.visitors.prev) || 0\n },\n visits: {\n value: parseInt(data.visits.value) || 0,\n prev: parseInt(data.visits.prev) || 0\n },\n bounces: {\n value: parseInt(data.bounces.value) || 0,\n prev: parseInt(data.bounces.prev) || 0\n },\n totaltime: {\n value: parseInt(data.totaltime.value) || 0,\n prev: parseInt(data.totaltime.prev) || 0\n }\n };\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "470715b6-0878-48b8-b6c6-40de27fbc966",
"name": "Send data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
1140,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Here is data from Umami analytics of Pennibnotes.com. Where X is URL and Y is number of visitors. Give me a table summary of this data in markdown format:{{ $('Parse Umami data').item.json.urlString }}.\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ea4bb37f-96d9-41b8-bf46-fb09865a6e0f",
"name": "Get page data from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
1380,
260
],
"parameters": {
"url": "=https://umami.rumjahn.synology.me/api/websites/f375d28c-1949-4597-8871-f1b942e3aa24/metrics?startAt={{Date.now() - (7 * 24 * 60 * 60 * 1000)}}&endAt={{Date.now()}}&type=url&tz=America/Los_Angeles",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4
},
{
"id": "d982606b-49c8-4d5b-ba79-bd0fdd2600b6",
"name": "Parse Umami data1",
"type": "n8n-nodes-base.code",
"position": [
1560,
260
],
"parameters": {
"jsCode": "// Get input data\nconst data = $input.all();\n\n// Create URL-encoded string from the data\nconst encodedData = encodeURIComponent(JSON.stringify(data));\n\n// Return the encoded data\nreturn {\n json: {\n thisWeek: encodedData\n }\n};"
},
"typeVersion": 2
},
{
"id": "f3734045-1318-4234-a3ac-61b766124609",
"name": "Get page view data from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
1760,
260
],
"parameters": {
"url": "=https://umami.rumjahn.synology.me/api/websites/f375d28c-1949-4597-8871-f1b942e3aa24/metrics?startAt={{Date.now() - (14 * 24 * 60 * 60 * 1000)}}&endAt={{Date.now() - (7 * 24 * 60 * 60 * 1000)}}&type=url&tz=America/Los_Angeles",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4
},
{
"id": "a0153ab0-3eaf-4f97-a2dc-ab63d45a9187",
"name": "Parse Umami",
"type": "n8n-nodes-base.code",
"position": [
1920,
260
],
"parameters": {
"jsCode": "// Get input data\nconst data = $input.all();\n\n// Create URL-encoded string from the data\nconst encodedData = encodeURIComponent(JSON.stringify(data));\n\n// Return the encoded data\nreturn {\n json: {\n lastweek: encodedData\n }\n};"
},
"typeVersion": 2
},
{
"id": "c2d3d396-09fa-4800-b56d-40ed7592cd3c",
"name": "Send data to A.I.1",
"type": "n8n-nodes-base.httpRequest",
"position": [
2180,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Here is data from Umami analytics of Pennibnotes.com. Where X is URL and Y is number of visitors. Compare the data from this week to last week. Present the data in a table using markdown and offer 5 improvement suggestions. This week:{{ $('Parse Umami data1').first().json.thisWeek }} Lastweek:{{ $json.lastweek }}\"\n }\n ]\n}\n\n",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ce58a556-c05a-4395-88b0-3edecbad80e5",
"name": "Save data to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
2520,
260
],
"parameters": {
"tableId": 607,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5870,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5869,
"fieldValue": "={{ $('Send data to A.I.').first().json.choices[0].message.content }}"
},
{
"fieldId": 5868,
"fieldValue": "={{ DateTime.now().toFormat('yyyy-MM-dd') }}"
},
{
"fieldId": 5871,
"fieldValue": "Name of your blog"
}
]
},
"operation": "create",
"databaseId": 121
},
"credentials": {
"baserowApi": {
"id": "8w0zXhycIfCAgja3",
"name": "Baserow account"
}
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "e28e067d-9245-4879-9321-4d21925f951e",
"connections": {
"Parse Umami": {
"main": [
[
{
"node": "Send data to A.I.1",
"type": "main",
"index": 0
}
]
]
},
"Parse Umami data": {
"main": [
[
{
"node": "Send data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get view stats from Umami",
"type": "main",
"index": 0
}
]
]
},
"Parse Umami data1": {
"main": [
[
{
"node": "Get page view data from Umami",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I.": {
"main": [
[
{
"node": "Get page data from Umami",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I.1": {
"main": [
[
{
"node": "Save data to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Get page data from Umami": {
"main": [
[
{
"node": "Parse Umami data1",
"type": "main",
"index": 0
}
]
]
},
"Get view stats from Umami": {
"main": [
[
{
"node": "Parse Umami data",
"type": "main",
"index": 0
}
]
]
},
"Get page view data from Umami": {
"main": [
[
{
"node": "Parse Umami",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get view stats from Umami",
"type": "main",
"index": 0
}
]
]
}
}
}

454
AI_Research_RAG_and_Data_Analysis/Summarize Umami data with AI (via Openrouter) and save it to Baserow.txt
View File

@@ -1,454 +0,0 @@
{
"id": "eZT6SZ4Kvmq5TzyQ",
"meta": {
"instanceId": "558d88703fb65b2d0e44613bc35916258b0f0bf983c5d4730c00c424b77ca36a",
"templateCredsSetupCompleted": true
},
"name": "Umami analytics template",
"tags": [],
"nodes": [
{
"id": "8a54ac1c-a072-42e6-a3ba-8cde33475eb5",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
460,
220
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e81c9be0-f59d-467e-9bda-eeb2d66ed31e",
"name": "Schedule Trigger",
"type": "n8n-nodes-base.scheduleTrigger",
"position": [
460,
380
],
"parameters": {
"rule": {
"interval": [
{
"field": "weeks",
"triggerAtDay": [
4
]
}
]
}
},
"typeVersion": 1.2
},
{
"id": "01b04872-9aea-4834-8df5-f6c91914133d",
"name": "Get view stats from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
760,
260
],
"parameters": {
"url": "=https://umami.mydomain.com/api/websites/86d4095c-a2a8-4fc8-9521-103e858e2b41/event-data/stats?startAt={{ DateTime.now().minus({ days: 7 }).toMillis() }}&endAt={{ DateTime.now().toMillis() }}&unit=hour&timezone=Asia%2FHong_Kong",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4.2
},
{
"id": "38d342e3-10ad-4260-8f44-5a3233ec3166",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
660,
-260
],
"parameters": {
"width": 504.88636363636317,
"content": "## Send data from Umami to A.I. and then save to Baserow\n\nYou can find out more about the stats available in the [Umami API](https://umami.is/docs/api/website-stats-api)\n\nRead the [case study here](https://rumjahn.com/how-to-analyze-umami-data-using-n8n-and-a-i-to-improve-seo-and-uncover-hidden-insights-for-better-content-optimization/).\n\n"
},
"typeVersion": 1
},
{
"id": "18c997fe-61b1-464a-8bb5-fcdc017dd1f6",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
660,
-60
],
"parameters": {
"color": 4,
"width": 393.16558441558414,
"height": 504.17207792207796,
"content": "## Get summary stats from Umami\n\nIt will get: Pageviews, Visitors, Visits, Bounces, Total Time\n\nYou need to change the URL to your website. https://{your website}/api/websites/{website ID}/\n\nYou can find your ID by going to your Umami account -> Settings -> Edit (next to domain)"
},
"typeVersion": 1
},
{
"id": "bfdc04a2-57fa-4a8a-b412-39047cebb370",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
1080,
-60
],
"parameters": {
"color": 5,
"width": 216.5746753246753,
"height": 502.37012987012963,
"content": "## Send data to A.I.\n\nTo use Openrouter, you need to register for an account.\nThen add header authorization credentials.\nUsername: Authroization\nPassword: Bearer {Your API Key}\n*It's Bearer space {API key}."
},
"typeVersion": 1
},
{
"id": "fc373fd7-52fc-4729-8022-021c09d0c89c",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
1320,
-60
],
"parameters": {
"color": 6,
"width": 746.3474025974022,
"height": 505.9740259740257,
"content": "## Get page specific stats for this week and last\n\nCalls Umami to get this week and last week's data. It will get the views for each page visited on your website for comparison."
},
"typeVersion": 1
},
{
"id": "82bd35b6-8b49-4d77-8be2-033a8bff3f41",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
-60
],
"parameters": {
"color": 5,
"width": 216.5746753246753,
"height": 502.37012987012963,
"content": "## Send data to A.I.\n\nTo use Openrouter, you need to register for an account.\nThen add header authorization credentials.\nUsername: Authroization\nPassword: Bearer {Your API Key}\n*It's Bearer space {API key}."
},
"typeVersion": 1
},
{
"id": "503c4ca3-36da-41a8-9029-f844a34daa59",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
2380,
-60
],
"parameters": {
"color": 4,
"width": 393.16558441558414,
"height": 504.17207792207796,
"content": "## Save analysis to baserow\n\nYou need to create a table in advance to save. \n- Date (date)\n- Summary (Long text)\n- Top pages (Long text)\n- Blog name (Long text)"
},
"typeVersion": 1
},
{
"id": "f64cdfbd-712f-461c-b025-25f37e2bded8",
"name": "Parse Umami data",
"type": "n8n-nodes-base.code",
"position": [
940,
260
],
"parameters": {
"jsCode": "function transformToUrlString(items) {\n // In n8n, we need to check if items is an array and get the json property\n const data = items[0].json;\n \n if (!data) {\n console.log('No valid data found');\n return encodeURIComponent(JSON.stringify([]));\n }\n \n try {\n // Create a simplified object with the metrics\n const simplified = {\n pageviews: {\n value: parseInt(data.pageviews.value) || 0,\n prev: parseInt(data.pageviews.prev) || 0\n },\n visitors: {\n value: parseInt(data.visitors.value) || 0,\n prev: parseInt(data.visitors.prev) || 0\n },\n visits: {\n value: parseInt(data.visits.value) || 0,\n prev: parseInt(data.visits.prev) || 0\n },\n bounces: {\n value: parseInt(data.bounces.value) || 0,\n prev: parseInt(data.bounces.prev) || 0\n },\n totaltime: {\n value: parseInt(data.totaltime.value) || 0,\n prev: parseInt(data.totaltime.prev) || 0\n }\n };\n \n return encodeURIComponent(JSON.stringify(simplified));\n } catch (error) {\n console.log('Error processing data:', error);\n throw new Error('Invalid data structure');\n }\n}\n\n// Get the input data\nconst items = $input.all();\n\n// Process the data\nconst result = transformToUrlString(items);\n\n// Return the result\nreturn { json: { urlString: result } };"
},
"typeVersion": 2
},
{
"id": "470715b6-0878-48b8-b6c6-40de27fbc966",
"name": "Send data to A.I.",
"type": "n8n-nodes-base.httpRequest",
"position": [
1140,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Here is data from Umami analytics of Pennibnotes.com. Where X is URL and Y is number of visitors. Give me a table summary of this data in markdown format:{{ $('Parse Umami data').item.json.urlString }}.\"\n }\n ]\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ea4bb37f-96d9-41b8-bf46-fb09865a6e0f",
"name": "Get page data from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
1380,
260
],
"parameters": {
"url": "=https://umami.rumjahn.synology.me/api/websites/f375d28c-1949-4597-8871-f1b942e3aa24/metrics?startAt={{Date.now() - (7 * 24 * 60 * 60 * 1000)}}&endAt={{Date.now()}}&type=url&tz=America/Los_Angeles",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4
},
{
"id": "d982606b-49c8-4d5b-ba79-bd0fdd2600b6",
"name": "Parse Umami data1",
"type": "n8n-nodes-base.code",
"position": [
1560,
260
],
"parameters": {
"jsCode": "// Get input data\nconst data = $input.all();\n\n// Create URL-encoded string from the data\nconst encodedData = encodeURIComponent(JSON.stringify(data));\n\n// Return the encoded data\nreturn {\n json: {\n thisWeek: encodedData\n }\n};"
},
"typeVersion": 2
},
{
"id": "f3734045-1318-4234-a3ac-61b766124609",
"name": "Get page view data from Umami",
"type": "n8n-nodes-base.httpRequest",
"position": [
1760,
260
],
"parameters": {
"url": "=https://umami.rumjahn.synology.me/api/websites/f375d28c-1949-4597-8871-f1b942e3aa24/metrics?startAt={{Date.now() - (14 * 24 * 60 * 60 * 1000)}}&endAt={{Date.now() - (7 * 24 * 60 * 60 * 1000)}}&type=url&tz=America/Los_Angeles",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "FKsKXvQUlaX5qt9n",
"name": "Header Auth account 3"
}
},
"typeVersion": 4
},
{
"id": "a0153ab0-3eaf-4f97-a2dc-ab63d45a9187",
"name": "Parse Umami",
"type": "n8n-nodes-base.code",
"position": [
1920,
260
],
"parameters": {
"jsCode": "// Get input data\nconst data = $input.all();\n\n// Create URL-encoded string from the data\nconst encodedData = encodeURIComponent(JSON.stringify(data));\n\n// Return the encoded data\nreturn {\n json: {\n lastweek: encodedData\n }\n};"
},
"typeVersion": 2
},
{
"id": "c2d3d396-09fa-4800-b56d-40ed7592cd3c",
"name": "Send data to A.I.1",
"type": "n8n-nodes-base.httpRequest",
"position": [
2180,
260
],
"parameters": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"method": "POST",
"options": {},
"jsonBody": "={\n \"model\": \"meta-llama/llama-3.1-70b-instruct:free\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"You are an SEO expert. Here is data from Umami analytics of Pennibnotes.com. Where X is URL and Y is number of visitors. Compare the data from this week to last week. Present the data in a table using markdown and offer 5 improvement suggestions. This week:{{ $('Parse Umami data1').first().json.thisWeek }} Lastweek:{{ $json.lastweek }}\"\n }\n ]\n}\n\n",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "WY7UkF14ksPKq3S8",
"name": "Header Auth account 2"
}
},
"typeVersion": 4.2
},
{
"id": "ce58a556-c05a-4395-88b0-3edecbad80e5",
"name": "Save data to Baserow",
"type": "n8n-nodes-base.baserow",
"position": [
2520,
260
],
"parameters": {
"tableId": 607,
"fieldsUi": {
"fieldValues": [
{
"fieldId": 5870,
"fieldValue": "={{ $json.choices[0].message.content }}"
},
{
"fieldId": 5869,
"fieldValue": "={{ $('Send data to A.I.').first().json.choices[0].message.content }}"
},
{
"fieldId": 5868,
"fieldValue": "={{ DateTime.now().toFormat('yyyy-MM-dd') }}"
},
{
"fieldId": 5871,
"fieldValue": "Name of your blog"
}
]
},
"operation": "create",
"databaseId": 121
},
"credentials": {
"baserowApi": {
"id": "8w0zXhycIfCAgja3",
"name": "Baserow account"
}
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "e28e067d-9245-4879-9321-4d21925f951e",
"connections": {
"Parse Umami": {
"main": [
[
{
"node": "Send data to A.I.1",
"type": "main",
"index": 0
}
]
]
},
"Parse Umami data": {
"main": [
[
{
"node": "Send data to A.I.",
"type": "main",
"index": 0
}
]
]
},
"Schedule Trigger": {
"main": [
[
{
"node": "Get view stats from Umami",
"type": "main",
"index": 0
}
]
]
},
"Parse Umami data1": {
"main": [
[
{
"node": "Get page view data from Umami",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I.": {
"main": [
[
{
"node": "Get page data from Umami",
"type": "main",
"index": 0
}
]
]
},
"Send data to A.I.1": {
"main": [
[
{
"node": "Save data to Baserow",
"type": "main",
"index": 0
}
]
]
},
"Get page data from Umami": {
"main": [
[
{
"node": "Parse Umami data1",
"type": "main",
"index": 0
}
]
]
},
"Get view stats from Umami": {
"main": [
[
{
"node": "Parse Umami data",
"type": "main",
"index": 0
}
]
]
},
"Get page view data from Umami": {
"main": [
[
{
"node": "Parse Umami",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Get view stats from Umami",
"type": "main",
"index": 0
}
]
]
}
}
}

1272
AI_Research_RAG_and_Data_Analysis/Survey Insights with Qdrant, Python and Information Extractor.json Normal file
View File

File diff suppressed because it is too large Load Diff

1272
AI_Research_RAG_and_Data_Analysis/Survey Insights with Qdrant, Python and Information Extractor.txt
View File

File diff suppressed because it is too large Load Diff

1971
AI_Research_RAG_and_Data_Analysis/Ultimate Scraper Workflow for n8n.json Normal file
View File

File diff suppressed because it is too large Load Diff

1971
AI_Research_RAG_and_Data_Analysis/Ultimate Scraper Workflow for n8n.txt
View File

File diff suppressed because it is too large Load Diff

688
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [1_3 anomaly][1_2 KNN].json Normal file
View File

@@ -0,0 +1,688 @@
{
"id": "pPtCy6qPfEv1qNRn",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[1/3 - anomaly detection] [1/2 - KNN classification] Batch upload dataset to Qdrant (crops dataset)",
"tags": [
{
"id": "n3zAUYFhdqtjhcLf",
"name": "qdrant",
"createdAt": "2024-12-10T11:56:59.987Z",
"updatedAt": "2024-12-10T11:56:59.987Z"
}
],
"nodes": [
{
"id": "53831410-b4f3-4374-8bdd-c2a33cd873cb",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-640,
0
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e303ccea-c0e0-4fe5-bd31-48380a0e438f",
"name": "Google Cloud Storage",
"type": "n8n-nodes-base.googleCloudStorage",
"position": [
820,
160
],
"parameters": {
"resource": "object",
"returnAll": true,
"bucketName": "n8n-qdrant-demo",
"listFilters": {
"prefix": "agricultural-crops"
},
"requestOptions": {}
},
"credentials": {
"googleCloudStorageOAuth2Api": {
"id": "fn0sr7grtfprVQvL",
"name": "Google Cloud Storage account"
}
},
"typeVersion": 1
},
{
"id": "737bdb15-61cf-48eb-96af-569eb5986ee8",
"name": "Get fields for Qdrant",
"type": "n8n-nodes-base.set",
"position": [
1080,
160
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "10d9147f-1c0c-4357-8413-3130829c2e24",
"name": "=publicLink",
"type": "string",
"value": "=https://storage.googleapis.com/{{ $json.bucket }}/{{ $json.selfLink.split('/').splice(-1) }}"
},
{
"id": "ff9e6a0b-e47a-4550-a13b-465507c75f8f",
"name": "cropName",
"type": "string",
"value": "={{ $json.id.split('/').slice(-3, -2)[0].toLowerCase()}}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "2b18ed0c-38d3-49e9-be3d-4f7b35f4d9e5",
"name": "Qdrant cluster variables",
"type": "n8n-nodes-base.set",
"position": [
-360,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "33581e0a-307f-4380-9533-615791096de7",
"name": "VoyageEmbeddingsDim",
"type": "number",
"value": 1024
},
{
"id": "6e390343-2cd2-4559-aba9-82b13acb7f52",
"name": "batchSize",
"type": "number",
"value": 4
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f88d290e-3311-4322-b2a5-1350fc1f8768",
"name": "Embed crop image",
"type": "n8n-nodes-base.httpRequest",
"position": [
2120,
160
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": $json.batchVoyage,\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "250c6a8d-f545-4037-8069-c834437bbe15",
"name": "Create Qdrant Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}",
"method": "PUT",
"options": {},
"jsonBody": "={{\n{\n \"vectors\": {\n \"voyage\": { \n \"size\": $('Qdrant cluster variables').first().json.VoyageEmbeddingsDim, \n \"distance\": \"Cosine\" \n } \n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "20b612ff-4794-43ef-bf45-008a16a2f30f",
"name": "Check Qdrant Collection Existence",
"type": "n8n-nodes-base.httpRequest",
"position": [
-100,
0
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/exists",
"options": {},
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "c067740b-5de3-452e-a614-bf14985a73a0",
"name": "Batches in the API's format",
"type": "n8n-nodes-base.set",
"position": [
1860,
160
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "f14db112-6f15-4405-aa47-8cb56bb8ae7a",
"name": "=batchVoyage",
"type": "array",
"value": "={{ $json.batch.map(item => ({ \"content\": ([{\"type\": \"image_url\", \"image_url\": item[\"publicLink\"]}])}))}}"
},
{
"id": "3885fd69-66f5-4435-86a4-b80eaa568ac1",
"name": "=batchPayloadQdrant",
"type": "array",
"value": "={{ $json.batch.map(item => ({\"crop_name\":item[\"cropName\"], \"image_path\":item[\"publicLink\"]})) }}"
},
{
"id": "8ea7a91e-af27-49cb-9a29-41dae15c4e33",
"name": "uuids",
"type": "array",
"value": "={{ $json.uuids }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "bf9a9532-db64-4c02-b91d-47e708ded4d3",
"name": "Batch Upload to Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
2320,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}/points",
"method": "PUT",
"options": {},
"jsonBody": "={{\n{\n \"batch\": {\n \"ids\" : $('Batches in the API\\'s format').item.json.uuids,\n \"vectors\": {\"voyage\": $json.data.map(item => item[\"embedding\"]) },\n \"payloads\": $('Batches in the API\\'s format').item.json.batchPayloadQdrant\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "3c30373f-c84c-405f-bb84-ec8b4c7419f4",
"name": "Split in batches, generate uuids for Qdrant points",
"type": "n8n-nodes-base.code",
"position": [
1600,
160
],
"parameters": {
"language": "python",
"pythonCode": "import uuid\n\ncrops = [item.json for item in _input.all()]\nbatch_size = int(_('Qdrant cluster variables').first()['json']['batchSize'])\n\ndef split_into_batches_add_uuids(array, batch_size):\n return [\n {\n \"batch\": array[i:i + batch_size],\n \"uuids\": [str(uuid.uuid4()) for j in range(len(array[i:i + batch_size]))]\n }\n for i in range(0, len(array), batch_size)\n ]\n\n# Split crops into batches\nbatched_crops = split_into_batches_add_uuids(crops, batch_size)\n\nreturn batched_crops"
},
"typeVersion": 2
},
{
"id": "2b028f8c-0a4c-4a3a-9e2b-14b1c2401c6d",
"name": "If collection exists",
"type": "n8n-nodes-base.if",
"position": [
120,
0
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "2104b862-667c-4a34-8888-9cb81a2e10f8",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
},
"leftValue": "={{ $json.result.exists }}",
"rightValue": "true"
}
]
}
},
"typeVersion": 2.2
},
{
"id": "768793f6-391e-4cc9-b637-f32ee2f77156",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
500,
340
],
"parameters": {
"width": 280,
"height": 200,
"content": "In the next workflow, we're going to use Qdrant to get the number of images belonging to each crop type defined by `crop_name` (for example, *\"cucumber\"*). \nTo get this information about counts in payload fields, we need to create an index on that field to optimise the resources (it needs to be done once). That's what is happening here"
},
"typeVersion": 1
},
{
"id": "0c8896f7-8c57-4add-bc4d-03c4a774bdf1",
"name": "Payload index on crop_name",
"type": "n8n-nodes-base.httpRequest",
"position": [
500,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}/index",
"method": "PUT",
"options": {},
"jsonBody": "={\n \"field_name\": \"crop_name\",\n \"field_schema\": \"keyword\"\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "342186f6-41bf-46be-9be8-a9b1ca290d55",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-360,
-360
],
"parameters": {
"height": 300,
"content": "Setting up variables\n1) Cloud URL - to connect to Qdrant Cloud (your personal cluster URL)\n2) Collection name in Qdrant\n3) Size of Voyage embeddings (needed for collection creation in Qdrant) <this one should not be changed unless the embedding model is changed>\n4) Batch size for batch embedding/batch uploading to Qdrant "
},
"typeVersion": 1
},
{
"id": "fae9248c-dbcc-4b6d-b977-0047f120a587",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-100,
-220
],
"parameters": {
"content": "In Qdrant, you can create a collection once; if you try to create it two times with the same name, you'll get an error, so I am adding here a check if a collection with this name exists already"
},
"typeVersion": 1
},
{
"id": "f7aea242-3d98-4a1c-a98a-986ac2b4928b",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
180,
340
],
"parameters": {
"height": 280,
"content": "If a collection with the name set up in variables doesn't exist yet, I create an empty one; \n\nCollection will contain [named vectors](https://qdrant.tech/documentation/concepts/vectors/#named-vectors), with a name *\"voyage\"*\nFor these named vectors, I define two parameters:\n1) Vectors size (in our case, Voyage embeddings size)\n2) Similarity metric to compare embeddings: in our case, **\"Cosine\"**.\n"
},
"typeVersion": 1
},
{
"id": "b84045c1-f66a-4543-8d42-1e76de0b6e91",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
800,
-280
],
"parameters": {
"height": 400,
"content": "Now it's time to embed & upload to Qdrant our image datasets;\nBoth of them, [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) and [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) were uploaded to our Google Cloud Storage bucket, and in this workflow we're fetching **the crops dataset** (for lands it will be a nearly identical workflow, up to variable names)\n(you should replace it with your image datasets)\n\nDatasets consist of **image URLs**; images are grouped by folders based on their class. For example, we have a system of subfolders like *\"tomato\"* and *\"cucumber\"* for the crops dataset with image URLs of the respective class.\n"
},
"typeVersion": 1
},
{
"id": "255dfad8-c545-4d75-bc9c-529aa50447a9",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
1080,
-140
],
"parameters": {
"height": 240,
"content": "Google Storage node returns **mediaLink**, which can be used directly for downloading images; however, we just need a public image URL so that Voyage API can process it; so here we construct this public link and extract a crop name from the folder in which image was stored (for example, *\"cucumber\"*)\n"
},
"typeVersion": 1
},
{
"id": "a6acce75-cce0-4de3-bc64-37592c97359b",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1600,
-80
],
"parameters": {
"height": 180,
"content": "I regroup images into batches of `batchSize` size and, to make batch upload to Qdrant possible, generate UUIDs to use them as batch [point IDs](https://qdrant.tech/documentation/concepts/points/#point-ids) (Qdrant doesn't set up id's for the user; users have to choose them themselves)"
},
"typeVersion": 1
},
{
"id": "cab3cc83-b50c-41f4-8d51-59e04bba5556",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1340,
-60
],
"parameters": {
"content": "Since we build anomaly detection based on the crops dataset, to test it properly, I didn't upload to Qdrant pictures of tomatoes at all; I filter them out here"
},
"typeVersion": 1
},
{
"id": "e5cdcce5-efdc-41f2-9796-656bd345f783",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
-100
],
"parameters": {
"height": 200,
"content": "Since Voyage API requires a [specific json structure](https://docs.voyageai.com/reference/multimodal-embeddings-api) for batch embeddings, as does [Qdrant's API for uploading points in batches](https://api.qdrant.tech/api-reference/points/upsert-points), I am adapting the structure of jsons\n\n[NB] - [payload = meta data in Qdrant]"
},
"typeVersion": 1
},
{
"id": "a7f15c44-3d5c-4b43-bfb2-94fe27a32071",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
20
],
"parameters": {
"width": 180,
"height": 80,
"content": "Embedding images with Voyage model (mind `input_type`)"
},
"typeVersion": 1
},
{
"id": "01b92e7e-d954-4d58-85b1-109c336546c4",
"name": "Filtering out tomato to test anomalies",
"type": "n8n-nodes-base.filter",
"position": [
1340,
160
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "f7953ae2-5333-4805-abe5-abf6da645c5e",
"operator": {
"type": "string",
"operation": "notEquals"
},
"leftValue": "={{ $json.cropName }}",
"rightValue": "tomato"
}
]
}
},
"typeVersion": 2.2
},
{
"id": "8d564817-885e-453a-a087-900b34b84d9c",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1160,
-280
],
"parameters": {
"width": 440,
"height": 460,
"content": "## Batch Uploading Dataset to Qdrant \n### This template imports dataset images from storage, creates embeddings for them in batches, and uploads them to Qdrant in batches. In this particular template, we work with [crops dataset](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification). However, it's analogous to [lands dataset](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification), and in general, it's adaptable to any dataset consisting of image URLs (as the following pipelines are).\n\n* First, check for an existing Qdrant collection to use; otherwise, create it here. Additionally, when creating the collection, we'll create a [payload index](https://qdrant.tech/documentation/concepts/indexing/#payload-index), which is required for a particular type of Qdrant requests we will use later.\n* Next, import all (dataset) images from Google Storage but keep only non-tomato-related ones (for anomaly detection testing).\n* Create (per batch) embeddings for all imported images using the Voyage AI multimodal embeddings API.\n* Finally, upload the resulting embeddings and image descriptors to Qdrant via batch uploading."
},
"typeVersion": 1
},
{
"id": "0233d3d0-bbdf-4d5b-a366-53cbfa4b6f9c",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-860,
360
],
"parameters": {
"color": 4,
"width": 540,
"height": 420,
"content": "### For anomaly detection\n**1. This is the first pipeline to upload (crops) dataset to Qdrant's collection.**\n2. The second pipeline is to set up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. The third is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.\n\n### For KNN (k nearest neighbours) classification\n**1. This is the first pipeline to upload (lands) dataset to Qdrant's collection.**\n2. The second is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.\n\n### To recreate both\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) and [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) datasets from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "27776c4a-3bf9-4704-9c13-345b75ffacc0",
"connections": {
"Embed crop image": {
"main": [
[
{
"node": "Batch Upload to Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Google Cloud Storage": {
"main": [
[
{
"node": "Get fields for Qdrant",
"type": "main",
"index": 0
}
]
]
},
"If collection exists": {
"main": [
[
{
"node": "Google Cloud Storage",
"type": "main",
"index": 0
}
],
[
{
"node": "Create Qdrant Collection",
"type": "main",
"index": 0
}
]
]
},
"Get fields for Qdrant": {
"main": [
[
{
"node": "Filtering out tomato to test anomalies",
"type": "main",
"index": 0
}
]
]
},
"Batch Upload to Qdrant": {
"main": [
[]
]
},
"Create Qdrant Collection": {
"main": [
[
{
"node": "Payload index on crop_name",
"type": "main",
"index": 0
}
]
]
},
"Qdrant cluster variables": {
"main": [
[
{
"node": "Check Qdrant Collection Existence",
"type": "main",
"index": 0
}
]
]
},
"Payload index on crop_name": {
"main": [
[
{
"node": "Google Cloud Storage",
"type": "main",
"index": 0
}
]
]
},
"Batches in the API's format": {
"main": [
[
{
"node": "Embed crop image",
"type": "main",
"index": 0
}
]
]
},
"Check Qdrant Collection Existence": {
"main": [
[
{
"node": "If collection exists",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Qdrant cluster variables",
"type": "main",
"index": 0
}
]
]
},
"Filtering out tomato to test anomalies": {
"main": [
[
{
"node": "Split in batches, generate uuids for Qdrant points",
"type": "main",
"index": 0
}
]
]
},
"Split in batches, generate uuids for Qdrant points": {
"main": [
[
{
"node": "Batches in the API's format",
"type": "main",
"index": 0
}
]
]
}
}
}

688
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [1_3 anomaly][1_2 KNN].txt
View File

@@ -1,688 +0,0 @@
{
"id": "pPtCy6qPfEv1qNRn",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[1/3 - anomaly detection] [1/2 - KNN classification] Batch upload dataset to Qdrant (crops dataset)",
"tags": [
{
"id": "n3zAUYFhdqtjhcLf",
"name": "qdrant",
"createdAt": "2024-12-10T11:56:59.987Z",
"updatedAt": "2024-12-10T11:56:59.987Z"
}
],
"nodes": [
{
"id": "53831410-b4f3-4374-8bdd-c2a33cd873cb",
"name": "When clicking ‘Test workflow’",
"type": "n8n-nodes-base.manualTrigger",
"position": [
-640,
0
],
"parameters": {},
"typeVersion": 1
},
{
"id": "e303ccea-c0e0-4fe5-bd31-48380a0e438f",
"name": "Google Cloud Storage",
"type": "n8n-nodes-base.googleCloudStorage",
"position": [
820,
160
],
"parameters": {
"resource": "object",
"returnAll": true,
"bucketName": "n8n-qdrant-demo",
"listFilters": {
"prefix": "agricultural-crops"
},
"requestOptions": {}
},
"credentials": {
"googleCloudStorageOAuth2Api": {
"id": "fn0sr7grtfprVQvL",
"name": "Google Cloud Storage account"
}
},
"typeVersion": 1
},
{
"id": "737bdb15-61cf-48eb-96af-569eb5986ee8",
"name": "Get fields for Qdrant",
"type": "n8n-nodes-base.set",
"position": [
1080,
160
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "10d9147f-1c0c-4357-8413-3130829c2e24",
"name": "=publicLink",
"type": "string",
"value": "=https://storage.googleapis.com/{{ $json.bucket }}/{{ $json.selfLink.split('/').splice(-1) }}"
},
{
"id": "ff9e6a0b-e47a-4550-a13b-465507c75f8f",
"name": "cropName",
"type": "string",
"value": "={{ $json.id.split('/').slice(-3, -2)[0].toLowerCase()}}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "2b18ed0c-38d3-49e9-be3d-4f7b35f4d9e5",
"name": "Qdrant cluster variables",
"type": "n8n-nodes-base.set",
"position": [
-360,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "33581e0a-307f-4380-9533-615791096de7",
"name": "VoyageEmbeddingsDim",
"type": "number",
"value": 1024
},
{
"id": "6e390343-2cd2-4559-aba9-82b13acb7f52",
"name": "batchSize",
"type": "number",
"value": 4
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f88d290e-3311-4322-b2a5-1350fc1f8768",
"name": "Embed crop image",
"type": "n8n-nodes-base.httpRequest",
"position": [
2120,
160
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": $json.batchVoyage,\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "250c6a8d-f545-4037-8069-c834437bbe15",
"name": "Create Qdrant Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
320,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}",
"method": "PUT",
"options": {},
"jsonBody": "={{\n{\n \"vectors\": {\n \"voyage\": { \n \"size\": $('Qdrant cluster variables').first().json.VoyageEmbeddingsDim, \n \"distance\": \"Cosine\" \n } \n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "20b612ff-4794-43ef-bf45-008a16a2f30f",
"name": "Check Qdrant Collection Existence",
"type": "n8n-nodes-base.httpRequest",
"position": [
-100,
0
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/exists",
"options": {},
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "c067740b-5de3-452e-a614-bf14985a73a0",
"name": "Batches in the API's format",
"type": "n8n-nodes-base.set",
"position": [
1860,
160
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "f14db112-6f15-4405-aa47-8cb56bb8ae7a",
"name": "=batchVoyage",
"type": "array",
"value": "={{ $json.batch.map(item => ({ \"content\": ([{\"type\": \"image_url\", \"image_url\": item[\"publicLink\"]}])}))}}"
},
{
"id": "3885fd69-66f5-4435-86a4-b80eaa568ac1",
"name": "=batchPayloadQdrant",
"type": "array",
"value": "={{ $json.batch.map(item => ({\"crop_name\":item[\"cropName\"], \"image_path\":item[\"publicLink\"]})) }}"
},
{
"id": "8ea7a91e-af27-49cb-9a29-41dae15c4e33",
"name": "uuids",
"type": "array",
"value": "={{ $json.uuids }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "bf9a9532-db64-4c02-b91d-47e708ded4d3",
"name": "Batch Upload to Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
2320,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}/points",
"method": "PUT",
"options": {},
"jsonBody": "={{\n{\n \"batch\": {\n \"ids\" : $('Batches in the API\\'s format').item.json.uuids,\n \"vectors\": {\"voyage\": $json.data.map(item => item[\"embedding\"]) },\n \"payloads\": $('Batches in the API\\'s format').item.json.batchPayloadQdrant\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "3c30373f-c84c-405f-bb84-ec8b4c7419f4",
"name": "Split in batches, generate uuids for Qdrant points",
"type": "n8n-nodes-base.code",
"position": [
1600,
160
],
"parameters": {
"language": "python",
"pythonCode": "import uuid\n\ncrops = [item.json for item in _input.all()]\nbatch_size = int(_('Qdrant cluster variables').first()['json']['batchSize'])\n\ndef split_into_batches_add_uuids(array, batch_size):\n return [\n {\n \"batch\": array[i:i + batch_size],\n \"uuids\": [str(uuid.uuid4()) for j in range(len(array[i:i + batch_size]))]\n }\n for i in range(0, len(array), batch_size)\n ]\n\n# Split crops into batches\nbatched_crops = split_into_batches_add_uuids(crops, batch_size)\n\nreturn batched_crops"
},
"typeVersion": 2
},
{
"id": "2b028f8c-0a4c-4a3a-9e2b-14b1c2401c6d",
"name": "If collection exists",
"type": "n8n-nodes-base.if",
"position": [
120,
0
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "2104b862-667c-4a34-8888-9cb81a2e10f8",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
},
"leftValue": "={{ $json.result.exists }}",
"rightValue": "true"
}
]
}
},
"typeVersion": 2.2
},
{
"id": "768793f6-391e-4cc9-b637-f32ee2f77156",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
500,
340
],
"parameters": {
"width": 280,
"height": 200,
"content": "In the next workflow, we're going to use Qdrant to get the number of images belonging to each crop type defined by `crop_name` (for example, *\"cucumber\"*). \nTo get this information about counts in payload fields, we need to create an index on that field to optimise the resources (it needs to be done once). That's what is happening here"
},
"typeVersion": 1
},
{
"id": "0c8896f7-8c57-4add-bc4d-03c4a774bdf1",
"name": "Payload index on crop_name",
"type": "n8n-nodes-base.httpRequest",
"position": [
500,
160
],
"parameters": {
"url": "={{ $('Qdrant cluster variables').first().json.qdrantCloudURL }}/collections/{{ $('Qdrant cluster variables').first().json.collectionName }}/index",
"method": "PUT",
"options": {},
"jsonBody": "={\n \"field_name\": \"crop_name\",\n \"field_schema\": \"keyword\"\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "342186f6-41bf-46be-9be8-a9b1ca290d55",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-360,
-360
],
"parameters": {
"height": 300,
"content": "Setting up variables\n1) Cloud URL - to connect to Qdrant Cloud (your personal cluster URL)\n2) Collection name in Qdrant\n3) Size of Voyage embeddings (needed for collection creation in Qdrant) <this one should not be changed unless the embedding model is changed>\n4) Batch size for batch embedding/batch uploading to Qdrant "
},
"typeVersion": 1
},
{
"id": "fae9248c-dbcc-4b6d-b977-0047f120a587",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-100,
-220
],
"parameters": {
"content": "In Qdrant, you can create a collection once; if you try to create it two times with the same name, you'll get an error, so I am adding here a check if a collection with this name exists already"
},
"typeVersion": 1
},
{
"id": "f7aea242-3d98-4a1c-a98a-986ac2b4928b",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
180,
340
],
"parameters": {
"height": 280,
"content": "If a collection with the name set up in variables doesn't exist yet, I create an empty one; \n\nCollection will contain [named vectors](https://qdrant.tech/documentation/concepts/vectors/#named-vectors), with a name *\"voyage\"*\nFor these named vectors, I define two parameters:\n1) Vectors size (in our case, Voyage embeddings size)\n2) Similarity metric to compare embeddings: in our case, **\"Cosine\"**.\n"
},
"typeVersion": 1
},
{
"id": "b84045c1-f66a-4543-8d42-1e76de0b6e91",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
800,
-280
],
"parameters": {
"height": 400,
"content": "Now it's time to embed & upload to Qdrant our image datasets;\nBoth of them, [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) and [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) were uploaded to our Google Cloud Storage bucket, and in this workflow we're fetching **the crops dataset** (for lands it will be a nearly identical workflow, up to variable names)\n(you should replace it with your image datasets)\n\nDatasets consist of **image URLs**; images are grouped by folders based on their class. For example, we have a system of subfolders like *\"tomato\"* and *\"cucumber\"* for the crops dataset with image URLs of the respective class.\n"
},
"typeVersion": 1
},
{
"id": "255dfad8-c545-4d75-bc9c-529aa50447a9",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
1080,
-140
],
"parameters": {
"height": 240,
"content": "Google Storage node returns **mediaLink**, which can be used directly for downloading images; however, we just need a public image URL so that Voyage API can process it; so here we construct this public link and extract a crop name from the folder in which image was stored (for example, *\"cucumber\"*)\n"
},
"typeVersion": 1
},
{
"id": "a6acce75-cce0-4de3-bc64-37592c97359b",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1600,
-80
],
"parameters": {
"height": 180,
"content": "I regroup images into batches of `batchSize` size and, to make batch upload to Qdrant possible, generate UUIDs to use them as batch [point IDs](https://qdrant.tech/documentation/concepts/points/#point-ids) (Qdrant doesn't set up id's for the user; users have to choose them themselves)"
},
"typeVersion": 1
},
{
"id": "cab3cc83-b50c-41f4-8d51-59e04bba5556",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
1340,
-60
],
"parameters": {
"content": "Since we build anomaly detection based on the crops dataset, to test it properly, I didn't upload to Qdrant pictures of tomatoes at all; I filter them out here"
},
"typeVersion": 1
},
{
"id": "e5cdcce5-efdc-41f2-9796-656bd345f783",
"name": "Sticky Note9",
"type": "n8n-nodes-base.stickyNote",
"position": [
1860,
-100
],
"parameters": {
"height": 200,
"content": "Since Voyage API requires a [specific json structure](https://docs.voyageai.com/reference/multimodal-embeddings-api) for batch embeddings, as does [Qdrant's API for uploading points in batches](https://api.qdrant.tech/api-reference/points/upsert-points), I am adapting the structure of jsons\n\n[NB] - [payload = meta data in Qdrant]"
},
"typeVersion": 1
},
{
"id": "a7f15c44-3d5c-4b43-bfb2-94fe27a32071",
"name": "Sticky Note11",
"type": "n8n-nodes-base.stickyNote",
"position": [
2120,
20
],
"parameters": {
"width": 180,
"height": 80,
"content": "Embedding images with Voyage model (mind `input_type`)"
},
"typeVersion": 1
},
{
"id": "01b92e7e-d954-4d58-85b1-109c336546c4",
"name": "Filtering out tomato to test anomalies",
"type": "n8n-nodes-base.filter",
"position": [
1340,
160
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "f7953ae2-5333-4805-abe5-abf6da645c5e",
"operator": {
"type": "string",
"operation": "notEquals"
},
"leftValue": "={{ $json.cropName }}",
"rightValue": "tomato"
}
]
}
},
"typeVersion": 2.2
},
{
"id": "8d564817-885e-453a-a087-900b34b84d9c",
"name": "Sticky Note8",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1160,
-280
],
"parameters": {
"width": 440,
"height": 460,
"content": "## Batch Uploading Dataset to Qdrant \n### This template imports dataset images from storage, creates embeddings for them in batches, and uploads them to Qdrant in batches. In this particular template, we work with [crops dataset](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification). However, it's analogous to [lands dataset](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification), and in general, it's adaptable to any dataset consisting of image URLs (as the following pipelines are).\n\n* First, check for an existing Qdrant collection to use; otherwise, create it here. Additionally, when creating the collection, we'll create a [payload index](https://qdrant.tech/documentation/concepts/indexing/#payload-index), which is required for a particular type of Qdrant requests we will use later.\n* Next, import all (dataset) images from Google Storage but keep only non-tomato-related ones (for anomaly detection testing).\n* Create (per batch) embeddings for all imported images using the Voyage AI multimodal embeddings API.\n* Finally, upload the resulting embeddings and image descriptors to Qdrant via batch uploading."
},
"typeVersion": 1
},
{
"id": "0233d3d0-bbdf-4d5b-a366-53cbfa4b6f9c",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-860,
360
],
"parameters": {
"color": 4,
"width": 540,
"height": 420,
"content": "### For anomaly detection\n**1. This is the first pipeline to upload (crops) dataset to Qdrant's collection.**\n2. The second pipeline is to set up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. The third is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.\n\n### For KNN (k nearest neighbours) classification\n**1. This is the first pipeline to upload (lands) dataset to Qdrant's collection.**\n2. The second is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.\n\n### To recreate both\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) and [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) datasets from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {},
"settings": {
"executionOrder": "v1"
},
"versionId": "27776c4a-3bf9-4704-9c13-345b75ffacc0",
"connections": {
"Embed crop image": {
"main": [
[
{
"node": "Batch Upload to Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Google Cloud Storage": {
"main": [
[
{
"node": "Get fields for Qdrant",
"type": "main",
"index": 0
}
]
]
},
"If collection exists": {
"main": [
[
{
"node": "Google Cloud Storage",
"type": "main",
"index": 0
}
],
[
{
"node": "Create Qdrant Collection",
"type": "main",
"index": 0
}
]
]
},
"Get fields for Qdrant": {
"main": [
[
{
"node": "Filtering out tomato to test anomalies",
"type": "main",
"index": 0
}
]
]
},
"Batch Upload to Qdrant": {
"main": [
[]
]
},
"Create Qdrant Collection": {
"main": [
[
{
"node": "Payload index on crop_name",
"type": "main",
"index": 0
}
]
]
},
"Qdrant cluster variables": {
"main": [
[
{
"node": "Check Qdrant Collection Existence",
"type": "main",
"index": 0
}
]
]
},
"Payload index on crop_name": {
"main": [
[
{
"node": "Google Cloud Storage",
"type": "main",
"index": 0
}
]
]
},
"Batches in the API's format": {
"main": [
[
{
"node": "Embed crop image",
"type": "main",
"index": 0
}
]
]
},
"Check Qdrant Collection Existence": {
"main": [
[
{
"node": "If collection exists",
"type": "main",
"index": 0
}
]
]
},
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Qdrant cluster variables",
"type": "main",
"index": 0
}
]
]
},
"Filtering out tomato to test anomalies": {
"main": [
[
{
"node": "Split in batches, generate uuids for Qdrant points",
"type": "main",
"index": 0
}
]
]
},
"Split in batches, generate uuids for Qdrant points": {
"main": [
[
{
"node": "Batches in the API's format",
"type": "main",
"index": 0
}
]
]
}
}
}

544
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_2 KNN] (1).json Normal file
View File

@@ -0,0 +1,544 @@
{
"id": "itzURpN5wbUNOXOw",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[2/2] KNN classifier (lands dataset)",
"tags": [
{
"id": "QN7etptCmdcGIpkS",
"name": "classifier",
"createdAt": "2024-12-08T22:08:15.968Z",
"updatedAt": "2024-12-09T19:25:04.113Z"
}
],
"nodes": [
{
"id": "33373ccb-164e-431c-8a9a-d68668fc70be",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
-140,
-240
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "58adecfa-45c7-4928-b850-053ea6f3b1c5",
"name": "Query Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
440,
-240
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.ImageEmbedding,\n \"using\": \"voyage\",\n \"limit\": $json.limitKNN,\n \"with_payload\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "258026b7-2dda-4165-bfe1-c4163b9caf78",
"name": "Majority Vote",
"type": "n8n-nodes-base.code",
"position": [
840,
-240
],
"parameters": {
"language": "python",
"pythonCode": "from collections import Counter\n\ninput_json = _input.all()[0]\npoints = input_json['json']['result']['points']\nmajority_vote_two_most_common = Counter([point[\"payload\"][\"landscape_name\"] for point in points]).most_common(2)\n\nreturn [{\n \"json\": {\n \"result\": majority_vote_two_most_common \n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "e83e7a0c-cb36-46d0-8908-86ee1bddf638",
"name": "Increase limitKNN",
"type": "n8n-nodes-base.set",
"position": [
1240,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "0b5d257b-1b27-48bc-bec2-78649bc844cc",
"name": "limitKNN",
"type": "number",
"value": "={{ $('Propagate loop variables').item.json.limitKNN + 5}}"
},
{
"id": "afee4bb3-f78b-4355-945d-3776e33337a4",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.ImageEmbedding }}"
},
{
"id": "701ed7ba-d112-4699-a611-c0c134757a6c",
"name": "qdrantCloudURL",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.qdrantCloudURL }}"
},
{
"id": "f5612f78-e7d8-4124-9c3a-27bd5870c9bf",
"name": "collectionName",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.collectionName }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "8edbff53-cba6-4491-9d5e-bac7ad6db418",
"name": "Propagate loop variables",
"type": "n8n-nodes-base.set",
"position": [
640,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "880838bf-2be2-4f5f-9417-974b3cbee163",
"name": "=limitKNN",
"type": "number",
"value": "={{ $json.result.points.length}}"
},
{
"id": "5fff2bea-f644-4fd9-ad04-afbecd19a5bc",
"name": "result",
"type": "object",
"value": "={{ $json.result }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "6fad4cc0-f02c-429d-aa4e-0d69ebab9d65",
"name": "Image Test URL",
"type": "n8n-nodes-base.set",
"position": [
-320,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f02e79e2-32c8-4af0-8bf9-281119b23cc0",
"name": "Return class",
"type": "n8n-nodes-base.set",
"position": [
1240,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bd8ca541-8758-4551-b667-1de373231364",
"name": "class",
"type": "string",
"value": "={{ $json.result[0][0] }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "83ca90fb-d5d5-45f4-8957-4363a4baf8ed",
"name": "Check tie",
"type": "n8n-nodes-base.if",
"position": [
1040,
-240
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "980663f6-9d7d-4e88-87b9-02030882472c",
"operator": {
"type": "number",
"operation": "gt"
},
"leftValue": "={{ $json.result.length }}",
"rightValue": 1
},
{
"id": "9f46fdeb-0f89-4010-99af-624c1c429d6a",
"operator": {
"type": "number",
"operation": "equals"
},
"leftValue": "={{ $json.result[0][1] }}",
"rightValue": "={{ $json.result[1][1] }}"
},
{
"id": "c59bc4fe-6821-4639-8595-fdaf4194c1e1",
"operator": {
"type": "number",
"operation": "lte"
},
"leftValue": "={{ $('Propagate loop variables').item.json.limitKNN }}",
"rightValue": 100
}
]
}
},
"typeVersion": 2.2
},
{
"id": "847ced21-4cfd-45d8-98fa-b578adc054d6",
"name": "Qdrant variables + embedding + KNN neigbours",
"type": "n8n-nodes-base.set",
"position": [
120,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de66070d-5e74-414e-8af7-d094cbc26f62",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
},
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=land-use"
},
{
"id": "db37e18d-340b-4624-84f6-df993af866d6",
"name": "limitKNN",
"type": "number",
"value": "=10"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "d1bc4edc-37d2-43ac-8d8b-560453e68d1f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-120
],
"parameters": {
"color": 6,
"width": 320,
"height": 540,
"content": "Here we're classifying existing types of satellite imagery of land types:\n- 'agricultural',\n- 'airplane',\n- 'baseballdiamond',\n- 'beach',\n- 'buildings',\n- 'chaparral',\n- 'denseresidential',\n- 'forest',\n- 'freeway',\n- 'golfcourse',\n- 'harbor',\n- 'intersection',\n- 'mediumresidential',\n- 'mobilehomepark',\n- 'overpass',\n- 'parkinglot',\n- 'river',\n- 'runway',\n- 'sparseresidential',\n- 'storagetanks',\n- 'tenniscourt'\n"
},
"typeVersion": 1
},
{
"id": "13560a31-3c72-43b8-9635-3f9ca11f23c9",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-460
],
"parameters": {
"color": 6,
"content": "I tested this KNN classifier on a whole `test` set of a dataset (it's not a part of the collection, only `validation` + `train` parts). Accuracy of classification on `test` is **93.24%**, no fine-tuning, no metric learning."
},
"typeVersion": 1
},
{
"id": "8c9dcbcb-a1ad-430f-b7dd-e19b5645b0f6",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-520,
-240
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b36fb270-2101-45e9-bb5c-06c4e07b769c",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1080,
-520
],
"parameters": {
"width": 460,
"height": 380,
"content": "## KNN classification workflow-tool\n### This n8n template takes an image URL (as anomaly detection tool does), and as output, it returns a class of the object on the image (out of land types list)\n\n* An image URL is received via the Execute Workflow Trigger, which is then sent to the Voyage.ai Multimodal Embeddings API to fetch its embedding.\n* The image's embedding vector is then used to query Qdrant, returning a set of X similar images with pre-labeled classes.\n* Majority voting is done for classes of neighbouring images.\n* A loop is used to resolve scenarios where there is a tie in Majority Voting (for example, we have 5 \"forest\" and 5 \"beach\"), and we increase the number of neighbours to retrieve.\n* When the loop finally resolves, the identified class is returned to the calling workflow."
},
"typeVersion": 1
},
{
"id": "51ece7fc-fd85-4d20-ae26-4df2d3893251",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
-40
],
"parameters": {
"height": 200,
"content": "Variables define another Qdrant's collection with landscapes (uploaded similarly as the crops collection, don't forget to switch it with your data) + amount of neighbours **limitKNN** in the database we'll use for an input image classification."
},
"typeVersion": 1
},
{
"id": "7aad5904-eb0b-4389-9d47-cc91780737ba",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-180,
-60
],
"parameters": {
"height": 80,
"content": "Similarly to anomaly detection tool, we're embedding input image with the Voyage model"
},
"typeVersion": 1
},
{
"id": "d3702707-ee4a-481f-82ca-d9386f5b7c8a",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-500
],
"parameters": {
"width": 740,
"height": 200,
"content": "## Tie loop\nHere we're [querying](https://api.qdrant.tech/api-reference/search/query-points) Qdrant, getting **limitKNN** nearest neighbours to our image <*Query Qdrant node*>, parsing their classes from payloads (images were pre-labeled & uploaded with their labels to Qdrant) & calculating the most frequent class name <*Majority Vote node*>. If there is a tie <*check tie node*> in 2 most common classes, for example, we have 5 \"forest\" and 5 \"harbor\", we repeat the procedure with the number of neighbours increased by 5 <*propagate loop variables node* and *increase limitKNN node*>.\nIf there is no tie, or we have already checked 100 neighbours, we exit the loop <*check tie node*> and return the class-answer."
},
"typeVersion": 1
},
{
"id": "d26911bb-0442-4adc-8511-7cec2d232393",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
160
],
"parameters": {
"height": 80,
"content": "Here, we extract the name of the input image class decided by the Majority Vote\n"
},
"typeVersion": 1
},
{
"id": "84ffc859-1d5c-4063-9051-3587f30a0017",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
80
],
"parameters": {
"color": 4,
"width": 540,
"height": 260,
"content": "### KNN (k nearest neighbours) classification\n1. The first pipeline is uploading (lands) dataset to Qdrant's collection.\n2. **This is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.**\n\n### To recreate it\nYou'll have to upload [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/land-use/images_train_test_val/test/buildings/buildings_000323.png"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "c8cfe732-fd78-4985-9540-ed8cb2de7ef3",
"connections": {
"Check tie": {
"main": [
[
{
"node": "Increase limitKNN",
"type": "main",
"index": 0
}
],
[
{
"node": "Return class",
"type": "main",
"index": 0
}
]
]
},
"Embed image": {
"main": [
[
{
"node": "Qdrant variables + embedding + KNN neigbours",
"type": "main",
"index": 0
}
]
]
},
"Query Qdrant": {
"main": [
[
{
"node": "Propagate loop variables",
"type": "main",
"index": 0
}
]
]
},
"Majority Vote": {
"main": [
[
{
"node": "Check tie",
"type": "main",
"index": 0
}
]
]
},
"Image Test URL": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
},
"Increase limitKNN": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image Test URL",
"type": "main",
"index": 0
}
]
]
},
"Propagate loop variables": {
"main": [
[
{
"node": "Majority Vote",
"type": "main",
"index": 0
}
]
]
},
"Qdrant variables + embedding + KNN neigbours": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
}
}
}

544
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_2 KNN] (1).txt
View File

@@ -1,544 +0,0 @@
{
"id": "itzURpN5wbUNOXOw",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[2/2] KNN classifier (lands dataset)",
"tags": [
{
"id": "QN7etptCmdcGIpkS",
"name": "classifier",
"createdAt": "2024-12-08T22:08:15.968Z",
"updatedAt": "2024-12-09T19:25:04.113Z"
}
],
"nodes": [
{
"id": "33373ccb-164e-431c-8a9a-d68668fc70be",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
-140,
-240
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "58adecfa-45c7-4928-b850-053ea6f3b1c5",
"name": "Query Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
440,
-240
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.ImageEmbedding,\n \"using\": \"voyage\",\n \"limit\": $json.limitKNN,\n \"with_payload\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "258026b7-2dda-4165-bfe1-c4163b9caf78",
"name": "Majority Vote",
"type": "n8n-nodes-base.code",
"position": [
840,
-240
],
"parameters": {
"language": "python",
"pythonCode": "from collections import Counter\n\ninput_json = _input.all()[0]\npoints = input_json['json']['result']['points']\nmajority_vote_two_most_common = Counter([point[\"payload\"][\"landscape_name\"] for point in points]).most_common(2)\n\nreturn [{\n \"json\": {\n \"result\": majority_vote_two_most_common \n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "e83e7a0c-cb36-46d0-8908-86ee1bddf638",
"name": "Increase limitKNN",
"type": "n8n-nodes-base.set",
"position": [
1240,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "0b5d257b-1b27-48bc-bec2-78649bc844cc",
"name": "limitKNN",
"type": "number",
"value": "={{ $('Propagate loop variables').item.json.limitKNN + 5}}"
},
{
"id": "afee4bb3-f78b-4355-945d-3776e33337a4",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.ImageEmbedding }}"
},
{
"id": "701ed7ba-d112-4699-a611-c0c134757a6c",
"name": "qdrantCloudURL",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.qdrantCloudURL }}"
},
{
"id": "f5612f78-e7d8-4124-9c3a-27bd5870c9bf",
"name": "collectionName",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.collectionName }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "8edbff53-cba6-4491-9d5e-bac7ad6db418",
"name": "Propagate loop variables",
"type": "n8n-nodes-base.set",
"position": [
640,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "880838bf-2be2-4f5f-9417-974b3cbee163",
"name": "=limitKNN",
"type": "number",
"value": "={{ $json.result.points.length}}"
},
{
"id": "5fff2bea-f644-4fd9-ad04-afbecd19a5bc",
"name": "result",
"type": "object",
"value": "={{ $json.result }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "6fad4cc0-f02c-429d-aa4e-0d69ebab9d65",
"name": "Image Test URL",
"type": "n8n-nodes-base.set",
"position": [
-320,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f02e79e2-32c8-4af0-8bf9-281119b23cc0",
"name": "Return class",
"type": "n8n-nodes-base.set",
"position": [
1240,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bd8ca541-8758-4551-b667-1de373231364",
"name": "class",
"type": "string",
"value": "={{ $json.result[0][0] }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "83ca90fb-d5d5-45f4-8957-4363a4baf8ed",
"name": "Check tie",
"type": "n8n-nodes-base.if",
"position": [
1040,
-240
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "980663f6-9d7d-4e88-87b9-02030882472c",
"operator": {
"type": "number",
"operation": "gt"
},
"leftValue": "={{ $json.result.length }}",
"rightValue": 1
},
{
"id": "9f46fdeb-0f89-4010-99af-624c1c429d6a",
"operator": {
"type": "number",
"operation": "equals"
},
"leftValue": "={{ $json.result[0][1] }}",
"rightValue": "={{ $json.result[1][1] }}"
},
{
"id": "c59bc4fe-6821-4639-8595-fdaf4194c1e1",
"operator": {
"type": "number",
"operation": "lte"
},
"leftValue": "={{ $('Propagate loop variables').item.json.limitKNN }}",
"rightValue": 100
}
]
}
},
"typeVersion": 2.2
},
{
"id": "847ced21-4cfd-45d8-98fa-b578adc054d6",
"name": "Qdrant variables + embedding + KNN neigbours",
"type": "n8n-nodes-base.set",
"position": [
120,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de66070d-5e74-414e-8af7-d094cbc26f62",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
},
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=land-use"
},
{
"id": "db37e18d-340b-4624-84f6-df993af866d6",
"name": "limitKNN",
"type": "number",
"value": "=10"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "d1bc4edc-37d2-43ac-8d8b-560453e68d1f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-120
],
"parameters": {
"color": 6,
"width": 320,
"height": 540,
"content": "Here we're classifying existing types of satellite imagery of land types:\n- 'agricultural',\n- 'airplane',\n- 'baseballdiamond',\n- 'beach',\n- 'buildings',\n- 'chaparral',\n- 'denseresidential',\n- 'forest',\n- 'freeway',\n- 'golfcourse',\n- 'harbor',\n- 'intersection',\n- 'mediumresidential',\n- 'mobilehomepark',\n- 'overpass',\n- 'parkinglot',\n- 'river',\n- 'runway',\n- 'sparseresidential',\n- 'storagetanks',\n- 'tenniscourt'\n"
},
"typeVersion": 1
},
{
"id": "13560a31-3c72-43b8-9635-3f9ca11f23c9",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-460
],
"parameters": {
"color": 6,
"content": "I tested this KNN classifier on a whole `test` set of a dataset (it's not a part of the collection, only `validation` + `train` parts). Accuracy of classification on `test` is **93.24%**, no fine-tuning, no metric learning."
},
"typeVersion": 1
},
{
"id": "8c9dcbcb-a1ad-430f-b7dd-e19b5645b0f6",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-520,
-240
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b36fb270-2101-45e9-bb5c-06c4e07b769c",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1080,
-520
],
"parameters": {
"width": 460,
"height": 380,
"content": "## KNN classification workflow-tool\n### This n8n template takes an image URL (as anomaly detection tool does), and as output, it returns a class of the object on the image (out of land types list)\n\n* An image URL is received via the Execute Workflow Trigger, which is then sent to the Voyage.ai Multimodal Embeddings API to fetch its embedding.\n* The image's embedding vector is then used to query Qdrant, returning a set of X similar images with pre-labeled classes.\n* Majority voting is done for classes of neighbouring images.\n* A loop is used to resolve scenarios where there is a tie in Majority Voting (for example, we have 5 \"forest\" and 5 \"beach\"), and we increase the number of neighbours to retrieve.\n* When the loop finally resolves, the identified class is returned to the calling workflow."
},
"typeVersion": 1
},
{
"id": "51ece7fc-fd85-4d20-ae26-4df2d3893251",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
-40
],
"parameters": {
"height": 200,
"content": "Variables define another Qdrant's collection with landscapes (uploaded similarly as the crops collection, don't forget to switch it with your data) + amount of neighbours **limitKNN** in the database we'll use for an input image classification."
},
"typeVersion": 1
},
{
"id": "7aad5904-eb0b-4389-9d47-cc91780737ba",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-180,
-60
],
"parameters": {
"height": 80,
"content": "Similarly to anomaly detection tool, we're embedding input image with the Voyage model"
},
"typeVersion": 1
},
{
"id": "d3702707-ee4a-481f-82ca-d9386f5b7c8a",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-500
],
"parameters": {
"width": 740,
"height": 200,
"content": "## Tie loop\nHere we're [querying](https://api.qdrant.tech/api-reference/search/query-points) Qdrant, getting **limitKNN** nearest neighbours to our image <*Query Qdrant node*>, parsing their classes from payloads (images were pre-labeled & uploaded with their labels to Qdrant) & calculating the most frequent class name <*Majority Vote node*>. If there is a tie <*check tie node*> in 2 most common classes, for example, we have 5 \"forest\" and 5 \"harbor\", we repeat the procedure with the number of neighbours increased by 5 <*propagate loop variables node* and *increase limitKNN node*>.\nIf there is no tie, or we have already checked 100 neighbours, we exit the loop <*check tie node*> and return the class-answer."
},
"typeVersion": 1
},
{
"id": "d26911bb-0442-4adc-8511-7cec2d232393",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
160
],
"parameters": {
"height": 80,
"content": "Here, we extract the name of the input image class decided by the Majority Vote\n"
},
"typeVersion": 1
},
{
"id": "84ffc859-1d5c-4063-9051-3587f30a0017",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
80
],
"parameters": {
"color": 4,
"width": 540,
"height": 260,
"content": "### KNN (k nearest neighbours) classification\n1. The first pipeline is uploading (lands) dataset to Qdrant's collection.\n2. **This is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.**\n\n### To recreate it\nYou'll have to upload [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/land-use/images_train_test_val/test/buildings/buildings_000323.png"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "c8cfe732-fd78-4985-9540-ed8cb2de7ef3",
"connections": {
"Check tie": {
"main": [
[
{
"node": "Increase limitKNN",
"type": "main",
"index": 0
}
],
[
{
"node": "Return class",
"type": "main",
"index": 0
}
]
]
},
"Embed image": {
"main": [
[
{
"node": "Qdrant variables + embedding + KNN neigbours",
"type": "main",
"index": 0
}
]
]
},
"Query Qdrant": {
"main": [
[
{
"node": "Propagate loop variables",
"type": "main",
"index": 0
}
]
]
},
"Majority Vote": {
"main": [
[
{
"node": "Check tie",
"type": "main",
"index": 0
}
]
]
},
"Image Test URL": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
},
"Increase limitKNN": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image Test URL",
"type": "main",
"index": 0
}
]
]
},
"Propagate loop variables": {
"main": [
[
{
"node": "Majority Vote",
"type": "main",
"index": 0
}
]
]
},
"Qdrant variables + embedding + KNN neigbours": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
}
}
}

544
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_2 KNN].json Normal file
View File

@@ -0,0 +1,544 @@
{
"id": "itzURpN5wbUNOXOw",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[2/2] KNN classifier (lands dataset)",
"tags": [
{
"id": "QN7etptCmdcGIpkS",
"name": "classifier",
"createdAt": "2024-12-08T22:08:15.968Z",
"updatedAt": "2024-12-09T19:25:04.113Z"
}
],
"nodes": [
{
"id": "33373ccb-164e-431c-8a9a-d68668fc70be",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
-140,
-240
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "58adecfa-45c7-4928-b850-053ea6f3b1c5",
"name": "Query Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
440,
-240
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.ImageEmbedding,\n \"using\": \"voyage\",\n \"limit\": $json.limitKNN,\n \"with_payload\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "258026b7-2dda-4165-bfe1-c4163b9caf78",
"name": "Majority Vote",
"type": "n8n-nodes-base.code",
"position": [
840,
-240
],
"parameters": {
"language": "python",
"pythonCode": "from collections import Counter\n\ninput_json = _input.all()[0]\npoints = input_json['json']['result']['points']\nmajority_vote_two_most_common = Counter([point[\"payload\"][\"landscape_name\"] for point in points]).most_common(2)\n\nreturn [{\n \"json\": {\n \"result\": majority_vote_two_most_common \n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "e83e7a0c-cb36-46d0-8908-86ee1bddf638",
"name": "Increase limitKNN",
"type": "n8n-nodes-base.set",
"position": [
1240,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "0b5d257b-1b27-48bc-bec2-78649bc844cc",
"name": "limitKNN",
"type": "number",
"value": "={{ $('Propagate loop variables').item.json.limitKNN + 5}}"
},
{
"id": "afee4bb3-f78b-4355-945d-3776e33337a4",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.ImageEmbedding }}"
},
{
"id": "701ed7ba-d112-4699-a611-c0c134757a6c",
"name": "qdrantCloudURL",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.qdrantCloudURL }}"
},
{
"id": "f5612f78-e7d8-4124-9c3a-27bd5870c9bf",
"name": "collectionName",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.collectionName }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "8edbff53-cba6-4491-9d5e-bac7ad6db418",
"name": "Propagate loop variables",
"type": "n8n-nodes-base.set",
"position": [
640,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "880838bf-2be2-4f5f-9417-974b3cbee163",
"name": "=limitKNN",
"type": "number",
"value": "={{ $json.result.points.length}}"
},
{
"id": "5fff2bea-f644-4fd9-ad04-afbecd19a5bc",
"name": "result",
"type": "object",
"value": "={{ $json.result }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "6fad4cc0-f02c-429d-aa4e-0d69ebab9d65",
"name": "Image Test URL",
"type": "n8n-nodes-base.set",
"position": [
-320,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f02e79e2-32c8-4af0-8bf9-281119b23cc0",
"name": "Return class",
"type": "n8n-nodes-base.set",
"position": [
1240,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bd8ca541-8758-4551-b667-1de373231364",
"name": "class",
"type": "string",
"value": "={{ $json.result[0][0] }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "83ca90fb-d5d5-45f4-8957-4363a4baf8ed",
"name": "Check tie",
"type": "n8n-nodes-base.if",
"position": [
1040,
-240
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "980663f6-9d7d-4e88-87b9-02030882472c",
"operator": {
"type": "number",
"operation": "gt"
},
"leftValue": "={{ $json.result.length }}",
"rightValue": 1
},
{
"id": "9f46fdeb-0f89-4010-99af-624c1c429d6a",
"operator": {
"type": "number",
"operation": "equals"
},
"leftValue": "={{ $json.result[0][1] }}",
"rightValue": "={{ $json.result[1][1] }}"
},
{
"id": "c59bc4fe-6821-4639-8595-fdaf4194c1e1",
"operator": {
"type": "number",
"operation": "lte"
},
"leftValue": "={{ $('Propagate loop variables').item.json.limitKNN }}",
"rightValue": 100
}
]
}
},
"typeVersion": 2.2
},
{
"id": "847ced21-4cfd-45d8-98fa-b578adc054d6",
"name": "Qdrant variables + embedding + KNN neigbours",
"type": "n8n-nodes-base.set",
"position": [
120,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de66070d-5e74-414e-8af7-d094cbc26f62",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
},
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=land-use"
},
{
"id": "db37e18d-340b-4624-84f6-df993af866d6",
"name": "limitKNN",
"type": "number",
"value": "=10"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "d1bc4edc-37d2-43ac-8d8b-560453e68d1f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-120
],
"parameters": {
"color": 6,
"width": 320,
"height": 540,
"content": "Here we're classifying existing types of satellite imagery of land types:\n- 'agricultural',\n- 'airplane',\n- 'baseballdiamond',\n- 'beach',\n- 'buildings',\n- 'chaparral',\n- 'denseresidential',\n- 'forest',\n- 'freeway',\n- 'golfcourse',\n- 'harbor',\n- 'intersection',\n- 'mediumresidential',\n- 'mobilehomepark',\n- 'overpass',\n- 'parkinglot',\n- 'river',\n- 'runway',\n- 'sparseresidential',\n- 'storagetanks',\n- 'tenniscourt'\n"
},
"typeVersion": 1
},
{
"id": "13560a31-3c72-43b8-9635-3f9ca11f23c9",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-460
],
"parameters": {
"color": 6,
"content": "I tested this KNN classifier on a whole `test` set of a dataset (it's not a part of the collection, only `validation` + `train` parts). Accuracy of classification on `test` is **93.24%**, no fine-tuning, no metric learning."
},
"typeVersion": 1
},
{
"id": "8c9dcbcb-a1ad-430f-b7dd-e19b5645b0f6",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-520,
-240
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b36fb270-2101-45e9-bb5c-06c4e07b769c",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1080,
-520
],
"parameters": {
"width": 460,
"height": 380,
"content": "## KNN classification workflow-tool\n### This n8n template takes an image URL (as anomaly detection tool does), and as output, it returns a class of the object on the image (out of land types list)\n\n* An image URL is received via the Execute Workflow Trigger, which is then sent to the Voyage.ai Multimodal Embeddings API to fetch its embedding.\n* The image's embedding vector is then used to query Qdrant, returning a set of X similar images with pre-labeled classes.\n* Majority voting is done for classes of neighbouring images.\n* A loop is used to resolve scenarios where there is a tie in Majority Voting (for example, we have 5 \"forest\" and 5 \"beach\"), and we increase the number of neighbours to retrieve.\n* When the loop finally resolves, the identified class is returned to the calling workflow."
},
"typeVersion": 1
},
{
"id": "51ece7fc-fd85-4d20-ae26-4df2d3893251",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
-40
],
"parameters": {
"height": 200,
"content": "Variables define another Qdrant's collection with landscapes (uploaded similarly as the crops collection, don't forget to switch it with your data) + amount of neighbours **limitKNN** in the database we'll use for an input image classification."
},
"typeVersion": 1
},
{
"id": "7aad5904-eb0b-4389-9d47-cc91780737ba",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-180,
-60
],
"parameters": {
"height": 80,
"content": "Similarly to anomaly detection tool, we're embedding input image with the Voyage model"
},
"typeVersion": 1
},
{
"id": "d3702707-ee4a-481f-82ca-d9386f5b7c8a",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-500
],
"parameters": {
"width": 740,
"height": 200,
"content": "## Tie loop\nHere we're [querying](https://api.qdrant.tech/api-reference/search/query-points) Qdrant, getting **limitKNN** nearest neighbours to our image <*Query Qdrant node*>, parsing their classes from payloads (images were pre-labeled & uploaded with their labels to Qdrant) & calculating the most frequent class name <*Majority Vote node*>. If there is a tie <*check tie node*> in 2 most common classes, for example, we have 5 \"forest\" and 5 \"harbor\", we repeat the procedure with the number of neighbours increased by 5 <*propagate loop variables node* and *increase limitKNN node*>.\nIf there is no tie, or we have already checked 100 neighbours, we exit the loop <*check tie node*> and return the class-answer."
},
"typeVersion": 1
},
{
"id": "d26911bb-0442-4adc-8511-7cec2d232393",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
160
],
"parameters": {
"height": 80,
"content": "Here, we extract the name of the input image class decided by the Majority Vote\n"
},
"typeVersion": 1
},
{
"id": "84ffc859-1d5c-4063-9051-3587f30a0017",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
80
],
"parameters": {
"color": 4,
"width": 540,
"height": 260,
"content": "### KNN (k nearest neighbours) classification\n1. The first pipeline is uploading (lands) dataset to Qdrant's collection.\n2. **This is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.**\n\n### To recreate it\nYou'll have to upload [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/land-use/images_train_test_val/test/buildings/buildings_000323.png"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "c8cfe732-fd78-4985-9540-ed8cb2de7ef3",
"connections": {
"Check tie": {
"main": [
[
{
"node": "Increase limitKNN",
"type": "main",
"index": 0
}
],
[
{
"node": "Return class",
"type": "main",
"index": 0
}
]
]
},
"Embed image": {
"main": [
[
{
"node": "Qdrant variables + embedding + KNN neigbours",
"type": "main",
"index": 0
}
]
]
},
"Query Qdrant": {
"main": [
[
{
"node": "Propagate loop variables",
"type": "main",
"index": 0
}
]
]
},
"Majority Vote": {
"main": [
[
{
"node": "Check tie",
"type": "main",
"index": 0
}
]
]
},
"Image Test URL": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
},
"Increase limitKNN": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image Test URL",
"type": "main",
"index": 0
}
]
]
},
"Propagate loop variables": {
"main": [
[
{
"node": "Majority Vote",
"type": "main",
"index": 0
}
]
]
},
"Qdrant variables + embedding + KNN neigbours": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
}
}
}

544
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_2 KNN].txt
View File

@@ -1,544 +0,0 @@
{
"id": "itzURpN5wbUNOXOw",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[2/2] KNN classifier (lands dataset)",
"tags": [
{
"id": "QN7etptCmdcGIpkS",
"name": "classifier",
"createdAt": "2024-12-08T22:08:15.968Z",
"updatedAt": "2024-12-09T19:25:04.113Z"
}
],
"nodes": [
{
"id": "33373ccb-164e-431c-8a9a-d68668fc70be",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
-140,
-240
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "58adecfa-45c7-4928-b850-053ea6f3b1c5",
"name": "Query Qdrant",
"type": "n8n-nodes-base.httpRequest",
"position": [
440,
-240
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.ImageEmbedding,\n \"using\": \"voyage\",\n \"limit\": $json.limitKNN,\n \"with_payload\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "258026b7-2dda-4165-bfe1-c4163b9caf78",
"name": "Majority Vote",
"type": "n8n-nodes-base.code",
"position": [
840,
-240
],
"parameters": {
"language": "python",
"pythonCode": "from collections import Counter\n\ninput_json = _input.all()[0]\npoints = input_json['json']['result']['points']\nmajority_vote_two_most_common = Counter([point[\"payload\"][\"landscape_name\"] for point in points]).most_common(2)\n\nreturn [{\n \"json\": {\n \"result\": majority_vote_two_most_common \n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "e83e7a0c-cb36-46d0-8908-86ee1bddf638",
"name": "Increase limitKNN",
"type": "n8n-nodes-base.set",
"position": [
1240,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "0b5d257b-1b27-48bc-bec2-78649bc844cc",
"name": "limitKNN",
"type": "number",
"value": "={{ $('Propagate loop variables').item.json.limitKNN + 5}}"
},
{
"id": "afee4bb3-f78b-4355-945d-3776e33337a4",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.ImageEmbedding }}"
},
{
"id": "701ed7ba-d112-4699-a611-c0c134757a6c",
"name": "qdrantCloudURL",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.qdrantCloudURL }}"
},
{
"id": "f5612f78-e7d8-4124-9c3a-27bd5870c9bf",
"name": "collectionName",
"type": "string",
"value": "={{ $('Qdrant variables + embedding + KNN neigbours').first().json.collectionName }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "8edbff53-cba6-4491-9d5e-bac7ad6db418",
"name": "Propagate loop variables",
"type": "n8n-nodes-base.set",
"position": [
640,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "880838bf-2be2-4f5f-9417-974b3cbee163",
"name": "=limitKNN",
"type": "number",
"value": "={{ $json.result.points.length}}"
},
{
"id": "5fff2bea-f644-4fd9-ad04-afbecd19a5bc",
"name": "result",
"type": "object",
"value": "={{ $json.result }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "6fad4cc0-f02c-429d-aa4e-0d69ebab9d65",
"name": "Image Test URL",
"type": "n8n-nodes-base.set",
"position": [
-320,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "f02e79e2-32c8-4af0-8bf9-281119b23cc0",
"name": "Return class",
"type": "n8n-nodes-base.set",
"position": [
1240,
0
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "bd8ca541-8758-4551-b667-1de373231364",
"name": "class",
"type": "string",
"value": "={{ $json.result[0][0] }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "83ca90fb-d5d5-45f4-8957-4363a4baf8ed",
"name": "Check tie",
"type": "n8n-nodes-base.if",
"position": [
1040,
-240
],
"parameters": {
"options": {},
"conditions": {
"options": {
"version": 2,
"leftValue": "",
"caseSensitive": true,
"typeValidation": "strict"
},
"combinator": "and",
"conditions": [
{
"id": "980663f6-9d7d-4e88-87b9-02030882472c",
"operator": {
"type": "number",
"operation": "gt"
},
"leftValue": "={{ $json.result.length }}",
"rightValue": 1
},
{
"id": "9f46fdeb-0f89-4010-99af-624c1c429d6a",
"operator": {
"type": "number",
"operation": "equals"
},
"leftValue": "={{ $json.result[0][1] }}",
"rightValue": "={{ $json.result[1][1] }}"
},
{
"id": "c59bc4fe-6821-4639-8595-fdaf4194c1e1",
"operator": {
"type": "number",
"operation": "lte"
},
"leftValue": "={{ $('Propagate loop variables').item.json.limitKNN }}",
"rightValue": 100
}
]
}
},
"typeVersion": 2.2
},
{
"id": "847ced21-4cfd-45d8-98fa-b578adc054d6",
"name": "Qdrant variables + embedding + KNN neigbours",
"type": "n8n-nodes-base.set",
"position": [
120,
-240
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "de66070d-5e74-414e-8af7-d094cbc26f62",
"name": "ImageEmbedding",
"type": "array",
"value": "={{ $json.data[0].embedding }}"
},
{
"id": "58b7384d-fd0c-44aa-9f8e-0306a99be431",
"name": "qdrantCloudURL",
"type": "string",
"value": "=https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "e34c4d88-b102-43cc-a09e-e0553f2da23a",
"name": "collectionName",
"type": "string",
"value": "=land-use"
},
{
"id": "db37e18d-340b-4624-84f6-df993af866d6",
"name": "limitKNN",
"type": "number",
"value": "=10"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "d1bc4edc-37d2-43ac-8d8b-560453e68d1f",
"name": "Sticky Note",
"type": "n8n-nodes-base.stickyNote",
"position": [
-940,
-120
],
"parameters": {
"color": 6,
"width": 320,
"height": 540,
"content": "Here we're classifying existing types of satellite imagery of land types:\n- 'agricultural',\n- 'airplane',\n- 'baseballdiamond',\n- 'beach',\n- 'buildings',\n- 'chaparral',\n- 'denseresidential',\n- 'forest',\n- 'freeway',\n- 'golfcourse',\n- 'harbor',\n- 'intersection',\n- 'mediumresidential',\n- 'mobilehomepark',\n- 'overpass',\n- 'parkinglot',\n- 'river',\n- 'runway',\n- 'sparseresidential',\n- 'storagetanks',\n- 'tenniscourt'\n"
},
"typeVersion": 1
},
{
"id": "13560a31-3c72-43b8-9635-3f9ca11f23c9",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
-460
],
"parameters": {
"color": 6,
"content": "I tested this KNN classifier on a whole `test` set of a dataset (it's not a part of the collection, only `validation` + `train` parts). Accuracy of classification on `test` is **93.24%**, no fine-tuning, no metric learning."
},
"typeVersion": 1
},
{
"id": "8c9dcbcb-a1ad-430f-b7dd-e19b5645b0f6",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-520,
-240
],
"parameters": {},
"typeVersion": 1
},
{
"id": "b36fb270-2101-45e9-bb5c-06c4e07b769c",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1080,
-520
],
"parameters": {
"width": 460,
"height": 380,
"content": "## KNN classification workflow-tool\n### This n8n template takes an image URL (as anomaly detection tool does), and as output, it returns a class of the object on the image (out of land types list)\n\n* An image URL is received via the Execute Workflow Trigger, which is then sent to the Voyage.ai Multimodal Embeddings API to fetch its embedding.\n* The image's embedding vector is then used to query Qdrant, returning a set of X similar images with pre-labeled classes.\n* Majority voting is done for classes of neighbouring images.\n* A loop is used to resolve scenarios where there is a tie in Majority Voting (for example, we have 5 \"forest\" and 5 \"beach\"), and we increase the number of neighbours to retrieve.\n* When the loop finally resolves, the identified class is returned to the calling workflow."
},
"typeVersion": 1
},
{
"id": "51ece7fc-fd85-4d20-ae26-4df2d3893251",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
120,
-40
],
"parameters": {
"height": 200,
"content": "Variables define another Qdrant's collection with landscapes (uploaded similarly as the crops collection, don't forget to switch it with your data) + amount of neighbours **limitKNN** in the database we'll use for an input image classification."
},
"typeVersion": 1
},
{
"id": "7aad5904-eb0b-4389-9d47-cc91780737ba",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-180,
-60
],
"parameters": {
"height": 80,
"content": "Similarly to anomaly detection tool, we're embedding input image with the Voyage model"
},
"typeVersion": 1
},
{
"id": "d3702707-ee4a-481f-82ca-d9386f5b7c8a",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
440,
-500
],
"parameters": {
"width": 740,
"height": 200,
"content": "## Tie loop\nHere we're [querying](https://api.qdrant.tech/api-reference/search/query-points) Qdrant, getting **limitKNN** nearest neighbours to our image <*Query Qdrant node*>, parsing their classes from payloads (images were pre-labeled & uploaded with their labels to Qdrant) & calculating the most frequent class name <*Majority Vote node*>. If there is a tie <*check tie node*> in 2 most common classes, for example, we have 5 \"forest\" and 5 \"harbor\", we repeat the procedure with the number of neighbours increased by 5 <*propagate loop variables node* and *increase limitKNN node*>.\nIf there is no tie, or we have already checked 100 neighbours, we exit the loop <*check tie node*> and return the class-answer."
},
"typeVersion": 1
},
{
"id": "d26911bb-0442-4adc-8511-7cec2d232393",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
1240,
160
],
"parameters": {
"height": 80,
"content": "Here, we extract the name of the input image class decided by the Majority Vote\n"
},
"typeVersion": 1
},
{
"id": "84ffc859-1d5c-4063-9051-3587f30a0017",
"name": "Sticky Note10",
"type": "n8n-nodes-base.stickyNote",
"position": [
-520,
80
],
"parameters": {
"color": 4,
"width": 540,
"height": 260,
"content": "### KNN (k nearest neighbours) classification\n1. The first pipeline is uploading (lands) dataset to Qdrant's collection.\n2. **This is the KNN classifier tool, which takes any image as input and classifies it based on queries to the Qdrant (lands) collection.**\n\n### To recreate it\nYou'll have to upload [lands](https://www.kaggle.com/datasets/apollo2506/landuse-scene-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/land-use/images_train_test_val/test/buildings/buildings_000323.png"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "c8cfe732-fd78-4985-9540-ed8cb2de7ef3",
"connections": {
"Check tie": {
"main": [
[
{
"node": "Increase limitKNN",
"type": "main",
"index": 0
}
],
[
{
"node": "Return class",
"type": "main",
"index": 0
}
]
]
},
"Embed image": {
"main": [
[
{
"node": "Qdrant variables + embedding + KNN neigbours",
"type": "main",
"index": 0
}
]
]
},
"Query Qdrant": {
"main": [
[
{
"node": "Propagate loop variables",
"type": "main",
"index": 0
}
]
]
},
"Majority Vote": {
"main": [
[
{
"node": "Check tie",
"type": "main",
"index": 0
}
]
]
},
"Image Test URL": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
},
"Increase limitKNN": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image Test URL",
"type": "main",
"index": 0
}
]
]
},
"Propagate loop variables": {
"main": [
[
{
"node": "Majority Vote",
"type": "main",
"index": 0
}
]
]
},
"Qdrant variables + embedding + KNN neigbours": {
"main": [
[
{
"node": "Query Qdrant",
"type": "main",
"index": 0
}
]
]
}
}
}

1250
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_3 - anomaly].json Normal file
View File

File diff suppressed because it is too large Load Diff

1250
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [2_3 - anomaly].txt
View File

File diff suppressed because it is too large Load Diff

461
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [3_3 - anomaly].json Normal file
View File

@@ -0,0 +1,461 @@
{
"id": "G8jRDBvwsMkkMiLN",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[3/3] Anomaly detection tool (crops dataset)",
"tags": [
{
"id": "spMntyrlE9ydvWFA",
"name": "anomaly-detection",
"createdAt": "2024-12-08T22:05:15.945Z",
"updatedAt": "2024-12-09T12:50:19.287Z"
}
],
"nodes": [
{
"id": "e01bafec-eb24-44c7-b3c4-a60f91666350",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
180
],
"parameters": {
"color": 6,
"width": 400,
"height": 740,
"content": "We are working here with crops dataset: \nExisting (so not anomalies) crops images in dataset are:\n- 'pearl_millet(bajra)',\n- 'tobacco-plant',\n- 'cherry',\n- 'cotton',\n- 'banana',\n- 'cucumber',\n- 'maize',\n- 'wheat',\n- 'clove',\n- 'jowar',\n- 'olive-tree',\n- 'soyabean',\n- 'coffee-plant',\n- 'rice',\n- 'lemon',\n- 'mustard-oil',\n- 'vigna-radiati(mung)',\n- 'coconut',\n- 'gram',\n- 'pineapple',\n- 'sugarcane',\n- 'sunflower',\n- 'chilli',\n- 'fox_nut(makhana)',\n- 'jute',\n- 'papaya',\n- 'tea',\n- 'cardamom',\n- 'almond'\n"
},
"typeVersion": 1
},
{
"id": "b9943781-de1f-4129-9b81-ed836e9ebb11",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
680,
60
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $('Image URL hardcode').first().json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "47b72bc2-4817-48c6-b517-c1328e402468",
"name": "Get similarity of medoids",
"type": "n8n-nodes-base.httpRequest",
"position": [
940,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.data[0].embedding,\n \"using\": \"voyage\",\n \"limit\": $('Info About Crop Labeled Clusters').first().json.cropsNumber,\n \"with_payload\": true,\n \"filter\": {\n \"must\": [\n { \n \"key\": $('Variables for medoids').first().json.clusterCenterType,\n \"match\": {\n \"value\": true\n }\n }\n ]\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "42d7eb27-ec38-4406-b5c4-27eb45358e93",
"name": "Compare scores",
"type": "n8n-nodes-base.code",
"position": [
1140,
60
],
"parameters": {
"language": "python",
"pythonCode": "points = _input.first()['json']['result']['points']\nthreshold_type = _('Variables for medoids').first()['json']['clusterThresholdCenterType']\n\nmax_score = -1\ncrop_with_max_score = None\nundefined = True\n\nfor center in points:\n if center['score'] >= center['payload'][threshold_type]:\n undefined = False\n if center['score'] > max_score:\n max_score = center['score']\n crop_with_max_score = center['payload']['crop_name']\n\nif undefined:\n result_message = \"ALERT, we might have a new undefined crop!\"\nelse:\n result_message = f\"Looks similar to {crop_with_max_score}\"\n\nreturn [{\n \"json\": {\n \"result\": result_message\n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "23aa604a-ff0b-4948-bcd5-af39300198c0",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
-220
],
"parameters": {
"width": 400,
"height": 380,
"content": "## Crop Anomaly Detection Tool\n### This is the tool that can be used directly for anomalous crops detection. \nIt takes as input (any) **image URL** and returns a **text message** telling if whatever this image depicts is anomalous to the crop dataset stored in Qdrant. \n\n* An Image URL is received via the Execute Workflow Trigger which is used to generate embedding vectors via the Voyage.ai Embeddings API.\n* The returned vectors are used to query the Qdrant collection to determine if the given crop is known by comparing it to **threshold scores** of each image class (crop type).\n* If the image scores lower than all thresholds, then the image is considered an anomaly for the dataset."
},
"typeVersion": 1
},
{
"id": "3a79eca2-44f9-4aee-8a0d-9c7ca2f9149d",
"name": "Variables for medoids",
"type": "n8n-nodes-base.set",
"position": [
-200,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "dbbc1e7b-c63e-4ff1-9524-8ef3e9f6cd48",
"name": "clusterCenterType",
"type": "string",
"value": "is_medoid"
},
{
"id": "a994ce37-2530-4030-acfb-ec777a7ddb05",
"name": "qdrantCloudURL",
"type": "string",
"value": "https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "12f0a9e6-686d-416e-a61b-72d034ec21ba",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "4c88a617-d44f-4776-b457-8a1dffb1d03c",
"name": "clusterThresholdCenterType",
"type": "string",
"value": "is_medoid_cluster_threshold"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "13b25434-bd66-4293-93f1-26c67b9ec7dd",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
260
],
"parameters": {
"color": 6,
"width": 360,
"height": 200,
"content": "**clusterCenterType** - either\n* \"is_text_anchor_medoid\" or\n* \"is_medoid\"\n\n\n**clusterThresholdCenterType** - either\n* \"is_text_anchor_medoid_cluster_threshold\" or\n* \"is_medoid_cluster_threshold\""
},
"typeVersion": 1
},
{
"id": "869b0962-6cae-487d-8230-539a0cc4c14c",
"name": "Info About Crop Labeled Clusters",
"type": "n8n-nodes-base.set",
"position": [
440,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5327b254-b703-4a34-a398-f82edb1d6d6b",
"name": "=cropsNumber",
"type": "number",
"value": "={{ $json.result.hits.length }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "5d3956f8-f43b-439e-b176-a594a21d8011",
"name": "Total Points in Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
40,
60
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/count",
"method": "POST",
"options": {},
"jsonBody": "={\n \"exact\": true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "14ba3db9-3965-4b20-b333-145616d45c3a",
"name": "Each Crop Counts",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/facet",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"key\": \"crop_name\",\n \"limit\": $json.result.count,\n \"exact\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "e37c6758-0556-4a56-ab14-d4df663cb53a",
"name": "Image URL hardcode",
"type": "n8n-nodes-base.set",
"position": [
-480,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "b24ad1a7-0cf8-4acc-9c18-6fe9d58b10f2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-720,
60
],
"parameters": {},
"typeVersion": 1
},
{
"id": "50424f2b-6831-41bf-8de4-81f69d901ce1",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-80
],
"parameters": {
"width": 180,
"height": 120,
"content": "Variables to access Qdrant's collection we uploaded & prepared for anomaly detection in 2 previous pipelines\n"
},
"typeVersion": 1
},
{
"id": "2e8ed3ca-1bba-4214-b34b-376a237842ff",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
-120
],
"parameters": {
"width": 560,
"height": 140,
"content": "These three nodes are needed just to figure out how many different classes (crops) we have in our Qdrant collection: **cropsNumber** (needed in *\"Get similarity of medoids\"* node. \n[Note] *\"Total Points in Collection\"* -> *\"Each Crop Counts\"* were used&explained already in *\"[2/4] Set up medoids (2 types) for anomaly detection (crops dataset)\"* pipeline.\n"
},
"typeVersion": 1
},
{
"id": "e2fa5763-6e97-4ff5-8919-1cb85a3c6968",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
240
],
"parameters": {
"height": 120,
"content": "Here, we're embedding the image passed to this workflow tool with the Voyage embedding model to compare the image to all crop images in the database."
},
"typeVersion": 1
},
{
"id": "cdb6b8d3-f7f4-4d66-850f-ce16c8ed98b9",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
920,
220
],
"parameters": {
"width": 400,
"height": 180,
"content": "Checking how similar the image is to all the centres of clusters (crops).\nIf it's more similar to the thresholds we set up and stored in centres in the previous workflow, the image probably belongs to this crop class; otherwise, it's anomalous to the class. \nIf image is anomalous to all the classes, it's an anomaly."
},
"typeVersion": 1
},
{
"id": "03b4699f-ba43-4f5f-ad69-6f81deea2641",
"name": "Sticky Note22",
"type": "n8n-nodes-base.stickyNote",
"position": [
-620,
580
],
"parameters": {
"color": 4,
"width": 540,
"height": 300,
"content": "### For anomaly detection\n1. The first pipeline is uploading (crops) dataset to Qdrant's collection.\n2. The second pipeline sets up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. **This is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.**\n\n### To recreate it\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/agricultural-crops%2Fcotton%2Fimage%20(36).jpg"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "f67b764b-9e1a-4db0-b9f2-490077a62f74",
"connections": {
"Embed image": {
"main": [
[
{
"node": "Get similarity of medoids",
"type": "main",
"index": 0
}
]
]
},
"Each Crop Counts": {
"main": [
[
{
"node": "Info About Crop Labeled Clusters",
"type": "main",
"index": 0
}
]
]
},
"Image URL hardcode": {
"main": [
[
{
"node": "Variables for medoids",
"type": "main",
"index": 0
}
]
]
},
"Variables for medoids": {
"main": [
[
{
"node": "Total Points in Collection",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image URL hardcode",
"type": "main",
"index": 0
}
]
]
},
"Get similarity of medoids": {
"main": [
[
{
"node": "Compare scores",
"type": "main",
"index": 0
}
]
]
},
"Total Points in Collection": {
"main": [
[
{
"node": "Each Crop Counts",
"type": "main",
"index": 0
}
]
]
},
"Info About Crop Labeled Clusters": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
}
}
}

461
AI_Research_RAG_and_Data_Analysis/Vector Database as a Big Data Analysis Tool for AI Agents [3_3 - anomaly].txt
View File

@@ -1,461 +0,0 @@
{
"id": "G8jRDBvwsMkkMiLN",
"meta": {
"instanceId": "205b3bc06c96f2dc835b4f00e1cbf9a937a74eeb3b47c99d0c30b0586dbf85aa"
},
"name": "[3/3] Anomaly detection tool (crops dataset)",
"tags": [
{
"id": "spMntyrlE9ydvWFA",
"name": "anomaly-detection",
"createdAt": "2024-12-08T22:05:15.945Z",
"updatedAt": "2024-12-09T12:50:19.287Z"
}
],
"nodes": [
{
"id": "e01bafec-eb24-44c7-b3c4-a60f91666350",
"name": "Sticky Note1",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
180
],
"parameters": {
"color": 6,
"width": 400,
"height": 740,
"content": "We are working here with crops dataset: \nExisting (so not anomalies) crops images in dataset are:\n- 'pearl_millet(bajra)',\n- 'tobacco-plant',\n- 'cherry',\n- 'cotton',\n- 'banana',\n- 'cucumber',\n- 'maize',\n- 'wheat',\n- 'clove',\n- 'jowar',\n- 'olive-tree',\n- 'soyabean',\n- 'coffee-plant',\n- 'rice',\n- 'lemon',\n- 'mustard-oil',\n- 'vigna-radiati(mung)',\n- 'coconut',\n- 'gram',\n- 'pineapple',\n- 'sugarcane',\n- 'sunflower',\n- 'chilli',\n- 'fox_nut(makhana)',\n- 'jute',\n- 'papaya',\n- 'tea',\n- 'cardamom',\n- 'almond'\n"
},
"typeVersion": 1
},
{
"id": "b9943781-de1f-4129-9b81-ed836e9ebb11",
"name": "Embed image",
"type": "n8n-nodes-base.httpRequest",
"position": [
680,
60
],
"parameters": {
"url": "https://api.voyageai.com/v1/multimodalembeddings",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"inputs\": [\n {\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": $('Image URL hardcode').first().json.imageURL\n }\n ]\n }\n ],\n \"model\": \"voyage-multimodal-3\",\n \"input_type\": \"document\"\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "Vb0RNVDnIHmgnZOP",
"name": "Voyage API"
}
},
"typeVersion": 4.2
},
{
"id": "47b72bc2-4817-48c6-b517-c1328e402468",
"name": "Get similarity of medoids",
"type": "n8n-nodes-base.httpRequest",
"position": [
940,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/points/query",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"query\": $json.data[0].embedding,\n \"using\": \"voyage\",\n \"limit\": $('Info About Crop Labeled Clusters').first().json.cropsNumber,\n \"with_payload\": true,\n \"filter\": {\n \"must\": [\n { \n \"key\": $('Variables for medoids').first().json.clusterCenterType,\n \"match\": {\n \"value\": true\n }\n }\n ]\n }\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "42d7eb27-ec38-4406-b5c4-27eb45358e93",
"name": "Compare scores",
"type": "n8n-nodes-base.code",
"position": [
1140,
60
],
"parameters": {
"language": "python",
"pythonCode": "points = _input.first()['json']['result']['points']\nthreshold_type = _('Variables for medoids').first()['json']['clusterThresholdCenterType']\n\nmax_score = -1\ncrop_with_max_score = None\nundefined = True\n\nfor center in points:\n if center['score'] >= center['payload'][threshold_type]:\n undefined = False\n if center['score'] > max_score:\n max_score = center['score']\n crop_with_max_score = center['payload']['crop_name']\n\nif undefined:\n result_message = \"ALERT, we might have a new undefined crop!\"\nelse:\n result_message = f\"Looks similar to {crop_with_max_score}\"\n\nreturn [{\n \"json\": {\n \"result\": result_message\n }\n}]\n"
},
"typeVersion": 2
},
{
"id": "23aa604a-ff0b-4948-bcd5-af39300198c0",
"name": "Sticky Note4",
"type": "n8n-nodes-base.stickyNote",
"position": [
-1200,
-220
],
"parameters": {
"width": 400,
"height": 380,
"content": "## Crop Anomaly Detection Tool\n### This is the tool that can be used directly for anomalous crops detection. \nIt takes as input (any) **image URL** and returns a **text message** telling if whatever this image depicts is anomalous to the crop dataset stored in Qdrant. \n\n* An Image URL is received via the Execute Workflow Trigger which is used to generate embedding vectors via the Voyage.ai Embeddings API.\n* The returned vectors are used to query the Qdrant collection to determine if the given crop is known by comparing it to **threshold scores** of each image class (crop type).\n* If the image scores lower than all thresholds, then the image is considered an anomaly for the dataset."
},
"typeVersion": 1
},
{
"id": "3a79eca2-44f9-4aee-8a0d-9c7ca2f9149d",
"name": "Variables for medoids",
"type": "n8n-nodes-base.set",
"position": [
-200,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "dbbc1e7b-c63e-4ff1-9524-8ef3e9f6cd48",
"name": "clusterCenterType",
"type": "string",
"value": "is_medoid"
},
{
"id": "a994ce37-2530-4030-acfb-ec777a7ddb05",
"name": "qdrantCloudURL",
"type": "string",
"value": "https://152bc6e2-832a-415c-a1aa-fb529f8baf8d.eu-central-1-0.aws.cloud.qdrant.io"
},
{
"id": "12f0a9e6-686d-416e-a61b-72d034ec21ba",
"name": "collectionName",
"type": "string",
"value": "=agricultural-crops"
},
{
"id": "4c88a617-d44f-4776-b457-8a1dffb1d03c",
"name": "clusterThresholdCenterType",
"type": "string",
"value": "is_medoid_cluster_threshold"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "13b25434-bd66-4293-93f1-26c67b9ec7dd",
"name": "Sticky Note3",
"type": "n8n-nodes-base.stickyNote",
"position": [
-340,
260
],
"parameters": {
"color": 6,
"width": 360,
"height": 200,
"content": "**clusterCenterType** - either\n* \"is_text_anchor_medoid\" or\n* \"is_medoid\"\n\n\n**clusterThresholdCenterType** - either\n* \"is_text_anchor_medoid_cluster_threshold\" or\n* \"is_medoid_cluster_threshold\""
},
"typeVersion": 1
},
{
"id": "869b0962-6cae-487d-8230-539a0cc4c14c",
"name": "Info About Crop Labeled Clusters",
"type": "n8n-nodes-base.set",
"position": [
440,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "5327b254-b703-4a34-a398-f82edb1d6d6b",
"name": "=cropsNumber",
"type": "number",
"value": "={{ $json.result.hits.length }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "5d3956f8-f43b-439e-b176-a594a21d8011",
"name": "Total Points in Collection",
"type": "n8n-nodes-base.httpRequest",
"position": [
40,
60
],
"parameters": {
"url": "={{ $json.qdrantCloudURL }}/collections/{{ $json.collectionName }}/points/count",
"method": "POST",
"options": {},
"jsonBody": "={\n \"exact\": true\n}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "14ba3db9-3965-4b20-b333-145616d45c3a",
"name": "Each Crop Counts",
"type": "n8n-nodes-base.httpRequest",
"position": [
240,
60
],
"parameters": {
"url": "={{ $('Variables for medoids').first().json.qdrantCloudURL }}/collections/{{ $('Variables for medoids').first().json.collectionName }}/facet",
"method": "POST",
"options": {},
"jsonBody": "={{\n{\n \"key\": \"crop_name\",\n \"limit\": $json.result.count,\n \"exact\": true\n}\n}}",
"sendBody": true,
"specifyBody": "json",
"authentication": "predefinedCredentialType",
"nodeCredentialType": "qdrantApi"
},
"credentials": {
"qdrantApi": {
"id": "it3j3hP9FICqhgX6",
"name": "QdrantApi account"
}
},
"typeVersion": 4.2
},
{
"id": "e37c6758-0556-4a56-ab14-d4df663cb53a",
"name": "Image URL hardcode",
"type": "n8n-nodes-base.set",
"position": [
-480,
60
],
"parameters": {
"options": {},
"assignments": {
"assignments": [
{
"id": "46ceba40-fb25-450c-8550-d43d8b8aa94c",
"name": "imageURL",
"type": "string",
"value": "={{ $json.query.imageURL }}"
}
]
}
},
"typeVersion": 3.4
},
{
"id": "b24ad1a7-0cf8-4acc-9c18-6fe9d58b10f2",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"position": [
-720,
60
],
"parameters": {},
"typeVersion": 1
},
{
"id": "50424f2b-6831-41bf-8de4-81f69d901ce1",
"name": "Sticky Note2",
"type": "n8n-nodes-base.stickyNote",
"position": [
-240,
-80
],
"parameters": {
"width": 180,
"height": 120,
"content": "Variables to access Qdrant's collection we uploaded & prepared for anomaly detection in 2 previous pipelines\n"
},
"typeVersion": 1
},
{
"id": "2e8ed3ca-1bba-4214-b34b-376a237842ff",
"name": "Sticky Note5",
"type": "n8n-nodes-base.stickyNote",
"position": [
40,
-120
],
"parameters": {
"width": 560,
"height": 140,
"content": "These three nodes are needed just to figure out how many different classes (crops) we have in our Qdrant collection: **cropsNumber** (needed in *\"Get similarity of medoids\"* node. \n[Note] *\"Total Points in Collection\"* -> *\"Each Crop Counts\"* were used&explained already in *\"[2/4] Set up medoids (2 types) for anomaly detection (crops dataset)\"* pipeline.\n"
},
"typeVersion": 1
},
{
"id": "e2fa5763-6e97-4ff5-8919-1cb85a3c6968",
"name": "Sticky Note6",
"type": "n8n-nodes-base.stickyNote",
"position": [
620,
240
],
"parameters": {
"height": 120,
"content": "Here, we're embedding the image passed to this workflow tool with the Voyage embedding model to compare the image to all crop images in the database."
},
"typeVersion": 1
},
{
"id": "cdb6b8d3-f7f4-4d66-850f-ce16c8ed98b9",
"name": "Sticky Note7",
"type": "n8n-nodes-base.stickyNote",
"position": [
920,
220
],
"parameters": {
"width": 400,
"height": 180,
"content": "Checking how similar the image is to all the centres of clusters (crops).\nIf it's more similar to the thresholds we set up and stored in centres in the previous workflow, the image probably belongs to this crop class; otherwise, it's anomalous to the class. \nIf image is anomalous to all the classes, it's an anomaly."
},
"typeVersion": 1
},
{
"id": "03b4699f-ba43-4f5f-ad69-6f81deea2641",
"name": "Sticky Note22",
"type": "n8n-nodes-base.stickyNote",
"position": [
-620,
580
],
"parameters": {
"color": 4,
"width": 540,
"height": 300,
"content": "### For anomaly detection\n1. The first pipeline is uploading (crops) dataset to Qdrant's collection.\n2. The second pipeline sets up cluster (class) centres in this Qdrant collection & cluster (class) threshold scores.\n3. **This is the anomaly detection tool, which takes any image as input and uses all preparatory work done with Qdrant (crops) collection.**\n\n### To recreate it\nYou'll have to upload [crops](https://www.kaggle.com/datasets/mdwaquarazam/agricultural-crops-image-classification) dataset from Kaggle to your own Google Storage bucket, and re-create APIs/connections to [Qdrant Cloud](https://qdrant.tech/documentation/quickstart-cloud/) (you can use **Free Tier** cluster), Voyage AI API & Google Cloud Storage\n\n**In general, pipelines are adaptable to any dataset of images**\n"
},
"typeVersion": 1
}
],
"active": false,
"pinData": {
"Execute Workflow Trigger": [
{
"json": {
"query": {
"imageURL": "https://storage.googleapis.com/n8n-qdrant-demo/agricultural-crops%2Fcotton%2Fimage%20(36).jpg"
}
}
}
]
},
"settings": {
"executionOrder": "v1"
},
"versionId": "f67b764b-9e1a-4db0-b9f2-490077a62f74",
"connections": {
"Embed image": {
"main": [
[
{
"node": "Get similarity of medoids",
"type": "main",
"index": 0
}
]
]
},
"Each Crop Counts": {
"main": [
[
{
"node": "Info About Crop Labeled Clusters",
"type": "main",
"index": 0
}
]
]
},
"Image URL hardcode": {
"main": [
[
{
"node": "Variables for medoids",
"type": "main",
"index": 0
}
]
]
},
"Variables for medoids": {
"main": [
[
{
"node": "Total Points in Collection",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "Image URL hardcode",
"type": "main",
"index": 0
}
]
]
},
"Get similarity of medoids": {
"main": [
[
{
"node": "Compare scores",
"type": "main",
"index": 0
}
]
]
},
"Total Points in Collection": {
"main": [
[
{
"node": "Each Crop Counts",
"type": "main",
"index": 0
}
]
]
},
"Info About Crop Labeled Clusters": {
"main": [
[
{
"node": "Embed image",
"type": "main",
"index": 0
}
]
]
}
}
}

1001
AI_Research_RAG_and_Data_Analysis/Visual Regression Testing with Apify and AI Vision Model.json Normal file
View File

File diff suppressed because it is too large Load Diff

1001
AI_Research_RAG_and_Data_Analysis/Visual Regression Testing with Apify and AI Vision Model.txt
View File

File diff suppressed because it is too large Load Diff

1389
AI_Research_RAG_and_Data_Analysis/🔍 Perplexity Research to HTML_ AI-Powered Content Creation.json Normal file
View File

File diff suppressed because one or more lines are too long

5553
AI_Research_RAG_and_Data_Analysis/🔍 Perplexity Research to HTML_ AI-Powered Content Creation.txt
View File

File diff suppressed because one or more lines are too long