Add Node.js implementation with enhanced search capabilities
- Implement complete Express.js server with SQLite FTS5 search - Add modern responsive UI with dark/light themes - Enhance search with partial word matching and advanced filters - Add RESTful API with comprehensive endpoints - Include security features (Helmet.js, rate limiting, CORS) - Add performance optimizations (gzip, caching, WAL mode) - Provide comprehensive documentation and setup scripts - Maintain feature parity with Python implementation while adding enhancements
This commit is contained in:
ivkan
605
src/database.js
Normal file
605
src/database.js
Normal file
@@ -0,0 +1,605 @@
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
const fs = require('fs-extra');
|
||||
const crypto = require('crypto');
|
||||
|
||||
class WorkflowDatabase {
|
||||
constructor(dbPath = 'database/workflows.db') {
|
||||
this.dbPath = dbPath;
|
||||
this.workflowsDir = 'workflows';
|
||||
this.db = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
if (this.initialized) return;
|
||||
await this.initDatabase();
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
async initDatabase() {
|
||||
// Ensure database directory exists
|
||||
const dbDir = path.dirname(this.dbPath);
|
||||
await fs.ensureDir(dbDir);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db = new sqlite3.Database(this.dbPath, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
// Enable WAL mode for better performance
|
||||
this.db.run('PRAGMA journal_mode=WAL');
|
||||
this.db.run('PRAGMA synchronous=NORMAL');
|
||||
this.db.run('PRAGMA cache_size=10000');
|
||||
this.db.run('PRAGMA temp_store=MEMORY');
|
||||
|
||||
this.createTables().then(resolve).catch(reject);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async createTables() {
|
||||
// Creating database tables
|
||||
return new Promise((resolve, reject) => {
|
||||
const queries = [
|
||||
// Main workflows table
|
||||
`CREATE TABLE IF NOT EXISTS workflows (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
filename TEXT UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
workflow_id TEXT,
|
||||
active BOOLEAN DEFAULT 0,
|
||||
description TEXT,
|
||||
trigger_type TEXT,
|
||||
complexity TEXT,
|
||||
node_count INTEGER DEFAULT 0,
|
||||
integrations TEXT,
|
||||
tags TEXT,
|
||||
created_at TEXT,
|
||||
updated_at TEXT,
|
||||
file_hash TEXT,
|
||||
file_size INTEGER,
|
||||
analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
|
||||
// FTS5 table for full-text search (simplified)
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS workflows_fts USING fts5(
|
||||
filename,
|
||||
name,
|
||||
description,
|
||||
integrations,
|
||||
tags
|
||||
)`,
|
||||
|
||||
// Indexes for performance
|
||||
'CREATE INDEX IF NOT EXISTS idx_trigger_type ON workflows(trigger_type)',
|
||||
'CREATE INDEX IF NOT EXISTS idx_complexity ON workflows(complexity)',
|
||||
'CREATE INDEX IF NOT EXISTS idx_active ON workflows(active)',
|
||||
'CREATE INDEX IF NOT EXISTS idx_node_count ON workflows(node_count)',
|
||||
'CREATE INDEX IF NOT EXISTS idx_filename ON workflows(filename)',
|
||||
|
||||
// Triggers to sync FTS table (simplified)
|
||||
`CREATE TRIGGER IF NOT EXISTS workflows_ai AFTER INSERT ON workflows BEGIN
|
||||
INSERT INTO workflows_fts(filename, name, description, integrations, tags)
|
||||
VALUES (new.filename, new.name, new.description, new.integrations, new.tags);
|
||||
END`,
|
||||
|
||||
`CREATE TRIGGER IF NOT EXISTS workflows_ad AFTER DELETE ON workflows BEGIN
|
||||
DELETE FROM workflows_fts WHERE filename = old.filename;
|
||||
END`,
|
||||
|
||||
`CREATE TRIGGER IF NOT EXISTS workflows_au AFTER UPDATE ON workflows BEGIN
|
||||
DELETE FROM workflows_fts WHERE filename = old.filename;
|
||||
INSERT INTO workflows_fts(filename, name, description, integrations, tags)
|
||||
VALUES (new.filename, new.name, new.description, new.integrations, new.tags);
|
||||
END`
|
||||
];
|
||||
|
||||
// Run queries sequentially to avoid race conditions
|
||||
const runQuery = (index) => {
|
||||
if (index >= queries.length) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const query = queries[index];
|
||||
this.db.run(query, (err) => {
|
||||
if (err) {
|
||||
console.error(`Error in query ${index + 1}:`, err.message);
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
runQuery(index + 1);
|
||||
});
|
||||
};
|
||||
|
||||
runQuery(0);
|
||||
});
|
||||
}
|
||||
|
||||
getFileHash(filePath) {
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
return crypto.createHash('md5').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
formatWorkflowName(filename) {
|
||||
// Remove .json extension and split by underscores
|
||||
const name = filename.replace('.json', '');
|
||||
const parts = name.split('_');
|
||||
|
||||
// Skip first part if it's just a number
|
||||
const startIndex = parts[0] && /^\d+$/.test(parts[0]) ? 1 : 0;
|
||||
const cleanParts = parts.slice(startIndex);
|
||||
|
||||
return cleanParts.map(part => {
|
||||
const lower = part.toLowerCase();
|
||||
const specialTerms = {
|
||||
'http': 'HTTP',
|
||||
'api': 'API',
|
||||
'webhook': 'Webhook',
|
||||
'automation': 'Automation',
|
||||
'automate': 'Automate',
|
||||
'scheduled': 'Scheduled',
|
||||
'triggered': 'Triggered',
|
||||
'manual': 'Manual'
|
||||
};
|
||||
|
||||
return specialTerms[lower] || part.charAt(0).toUpperCase() + part.slice(1);
|
||||
}).join(' ');
|
||||
}
|
||||
|
||||
analyzeWorkflow(filePath) {
|
||||
try {
|
||||
const data = fs.readJsonSync(filePath);
|
||||
const filename = path.basename(filePath);
|
||||
const fileSize = fs.statSync(filePath).size;
|
||||
const fileHash = this.getFileHash(filePath);
|
||||
|
||||
const workflow = {
|
||||
filename,
|
||||
name: this.formatWorkflowName(filename),
|
||||
workflow_id: data.id || '',
|
||||
active: data.active || false,
|
||||
nodes: data.nodes || [],
|
||||
connections: data.connections || {},
|
||||
tags: data.tags || [],
|
||||
created_at: data.createdAt || '',
|
||||
updated_at: data.updatedAt || '',
|
||||
file_hash: fileHash,
|
||||
file_size: fileSize
|
||||
};
|
||||
|
||||
// Use meaningful JSON name if available
|
||||
const jsonName = data.name?.trim();
|
||||
if (jsonName && jsonName !== filename.replace('.json', '') && !jsonName.startsWith('My workflow')) {
|
||||
workflow.name = jsonName;
|
||||
}
|
||||
|
||||
// Analyze nodes
|
||||
const nodeCount = workflow.nodes.length;
|
||||
workflow.node_count = nodeCount;
|
||||
|
||||
// Determine complexity
|
||||
if (nodeCount <= 5) {
|
||||
workflow.complexity = 'low';
|
||||
} else if (nodeCount <= 15) {
|
||||
workflow.complexity = 'medium';
|
||||
} else {
|
||||
workflow.complexity = 'high';
|
||||
}
|
||||
|
||||
// Analyze trigger type and integrations
|
||||
const { triggerType, integrations } = this.analyzeNodes(workflow.nodes);
|
||||
workflow.trigger_type = triggerType;
|
||||
workflow.integrations = Array.from(integrations);
|
||||
|
||||
// Generate description
|
||||
workflow.description = this.generateDescription(workflow, triggerType, integrations);
|
||||
|
||||
return workflow;
|
||||
} catch (error) {
|
||||
console.error(`Error analyzing workflow ${filePath}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
analyzeNodes(nodes) {
|
||||
const integrations = new Set();
|
||||
let triggerType = 'Manual';
|
||||
|
||||
nodes.forEach(node => {
|
||||
const nodeType = node.type || '';
|
||||
|
||||
// Extract integration name from node type
|
||||
if (nodeType.includes('.')) {
|
||||
const parts = nodeType.split('.');
|
||||
if (parts.length >= 2) {
|
||||
const integration = parts[1];
|
||||
if (integration !== 'core' && integration !== 'base') {
|
||||
integrations.add(integration.charAt(0).toUpperCase() + integration.slice(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine trigger type based on node types
|
||||
if (nodeType.includes('webhook')) {
|
||||
triggerType = 'Webhook';
|
||||
} else if (nodeType.includes('cron') || nodeType.includes('schedule')) {
|
||||
triggerType = 'Scheduled';
|
||||
} else if (nodeType.includes('trigger')) {
|
||||
triggerType = 'Triggered';
|
||||
}
|
||||
});
|
||||
|
||||
return { triggerType, integrations };
|
||||
}
|
||||
|
||||
generateDescription(workflow, triggerType, integrations) {
|
||||
const parts = [];
|
||||
|
||||
// Add trigger info
|
||||
if (triggerType !== 'Manual') {
|
||||
parts.push(`${triggerType} workflow`);
|
||||
} else {
|
||||
parts.push('Manual workflow');
|
||||
}
|
||||
|
||||
// Add integration info
|
||||
if (integrations.size > 0) {
|
||||
const integrationList = Array.from(integrations).slice(0, 3);
|
||||
if (integrations.size > 3) {
|
||||
integrationList.push(`+${integrations.size - 3} more`);
|
||||
}
|
||||
parts.push(`integrating ${integrationList.join(', ')}`);
|
||||
}
|
||||
|
||||
// Add complexity info
|
||||
parts.push(`with ${workflow.node_count} nodes (${workflow.complexity} complexity)`);
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
async indexWorkflows(forceReindex = false) {
|
||||
if (!this.initialized) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
const workflowFiles = await fs.readdir(this.workflowsDir);
|
||||
const jsonFiles = workflowFiles.filter(file => file.endsWith('.json'));
|
||||
|
||||
let processed = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const file of jsonFiles) {
|
||||
const filePath = path.join(this.workflowsDir, file);
|
||||
const workflow = this.analyzeWorkflow(filePath);
|
||||
|
||||
if (!workflow) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if workflow exists and if hash changed
|
||||
const existing = await this.getWorkflowByFilename(file);
|
||||
if (!forceReindex && existing && existing.file_hash === workflow.file_hash) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.upsertWorkflow(workflow);
|
||||
processed++;
|
||||
} catch (error) {
|
||||
console.error(`Error indexing workflow ${file}:`, error.message);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
return { processed, skipped, errors, total: jsonFiles.length };
|
||||
}
|
||||
|
||||
async getWorkflowByFilename(filename) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(
|
||||
'SELECT * FROM workflows WHERE filename = ?',
|
||||
[filename],
|
||||
(err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async upsertWorkflow(workflow) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const sql = `
|
||||
INSERT OR REPLACE INTO workflows (
|
||||
filename, name, workflow_id, active, description, trigger_type,
|
||||
complexity, node_count, integrations, tags, created_at, updated_at,
|
||||
file_hash, file_size, analyzed_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
`;
|
||||
|
||||
const params = [
|
||||
workflow.filename,
|
||||
workflow.name,
|
||||
workflow.workflow_id,
|
||||
workflow.active,
|
||||
workflow.description,
|
||||
workflow.trigger_type,
|
||||
workflow.complexity,
|
||||
workflow.node_count,
|
||||
JSON.stringify(workflow.integrations),
|
||||
JSON.stringify(workflow.tags),
|
||||
workflow.created_at,
|
||||
workflow.updated_at,
|
||||
workflow.file_hash,
|
||||
workflow.file_size
|
||||
];
|
||||
|
||||
this.db.run(sql, params, function(err) {
|
||||
if (err) reject(err);
|
||||
else resolve(this.lastID);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
buildFTSQuery(query) {
|
||||
// Escape FTS5 special characters and build partial matching query
|
||||
let cleanQuery = query
|
||||
.replace(/[^\w\s"'-]/g, ' ') // Remove special chars except quotes, hyphens, apostrophes
|
||||
.trim();
|
||||
|
||||
if (!cleanQuery) return '*';
|
||||
|
||||
// Handle quoted phrases
|
||||
const phrases = [];
|
||||
const quotedRegex = /"([^"]+)"/g;
|
||||
let match;
|
||||
|
||||
while ((match = quotedRegex.exec(cleanQuery)) !== null) {
|
||||
phrases.push(`"${match[1]}"`); // Keep exact phrases
|
||||
cleanQuery = cleanQuery.replace(match[0], ' ');
|
||||
}
|
||||
|
||||
// Split remaining terms and add wildcards for partial matching
|
||||
const terms = cleanQuery
|
||||
.split(/\s+/)
|
||||
.filter(term => term.length > 0)
|
||||
.map(term => {
|
||||
// Add wildcard suffix for prefix matching
|
||||
if (term.length >= 2) {
|
||||
return `${term}*`;
|
||||
}
|
||||
return term;
|
||||
});
|
||||
|
||||
// Combine phrases and wildcard terms
|
||||
const allTerms = [...phrases, ...terms];
|
||||
|
||||
if (allTerms.length === 0) return '*';
|
||||
|
||||
// Join with AND for more precise results
|
||||
return allTerms.join(' AND ');
|
||||
}
|
||||
|
||||
async searchWorkflows(query = '', triggerFilter = 'all', complexityFilter = 'all',
|
||||
activeOnly = false, limit = 50, offset = 0) {
|
||||
if (!this.initialized) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let sql = '';
|
||||
let params = [];
|
||||
|
||||
if (query.trim()) {
|
||||
// Use FTS search with partial matching
|
||||
const ftsQuery = this.buildFTSQuery(query.trim());
|
||||
sql = `
|
||||
SELECT w.* FROM workflows w
|
||||
JOIN workflows_fts fts ON w.id = fts.rowid
|
||||
WHERE workflows_fts MATCH ?
|
||||
`;
|
||||
params.push(ftsQuery);
|
||||
} else {
|
||||
// Regular search
|
||||
sql = 'SELECT * FROM workflows WHERE 1=1';
|
||||
}
|
||||
|
||||
// Add filters
|
||||
if (triggerFilter !== 'all') {
|
||||
sql += ' AND trigger_type = ?';
|
||||
params.push(triggerFilter);
|
||||
}
|
||||
|
||||
if (complexityFilter !== 'all') {
|
||||
sql += ' AND complexity = ?';
|
||||
params.push(complexityFilter);
|
||||
}
|
||||
|
||||
if (activeOnly) {
|
||||
sql += ' AND active = 1';
|
||||
}
|
||||
|
||||
// Count total - rebuild query for FTS compatibility
|
||||
let countSql;
|
||||
let countParams = [...params];
|
||||
|
||||
if (query.trim()) {
|
||||
// For FTS queries, we need to rebuild the count query
|
||||
countSql = `
|
||||
SELECT COUNT(*) as total FROM workflows w
|
||||
JOIN workflows_fts fts ON w.id = fts.rowid
|
||||
WHERE workflows_fts MATCH ?
|
||||
`;
|
||||
countParams = [this.buildFTSQuery(query.trim())];
|
||||
|
||||
// Add filters to count query
|
||||
if (triggerFilter !== 'all') {
|
||||
countSql += ' AND trigger_type = ?';
|
||||
countParams.push(triggerFilter);
|
||||
}
|
||||
|
||||
if (complexityFilter !== 'all') {
|
||||
countSql += ' AND complexity = ?';
|
||||
countParams.push(complexityFilter);
|
||||
}
|
||||
|
||||
if (activeOnly) {
|
||||
countSql += ' AND active = 1';
|
||||
}
|
||||
} else {
|
||||
countSql = `SELECT COUNT(*) as total FROM (${sql})`;
|
||||
countParams = params.slice(0, -2); // Remove LIMIT and OFFSET for count
|
||||
}
|
||||
|
||||
this.db.get(countSql, countParams, (err, countResult) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const total = countResult.total;
|
||||
|
||||
// Add pagination
|
||||
sql += ' ORDER BY name LIMIT ? OFFSET ?';
|
||||
params.push(limit, offset);
|
||||
|
||||
this.db.all(sql, params, (err, rows) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse JSON fields
|
||||
const workflows = rows.map(row => ({
|
||||
...row,
|
||||
integrations: JSON.parse(row.integrations || '[]'),
|
||||
tags: JSON.parse(row.tags || '[]')
|
||||
}));
|
||||
|
||||
resolve({ workflows, total });
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async getStats() {
|
||||
if (!this.initialized) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const queries = [
|
||||
'SELECT COUNT(*) as total FROM workflows',
|
||||
'SELECT COUNT(*) as active FROM workflows WHERE active = 1',
|
||||
'SELECT COUNT(*) as inactive FROM workflows WHERE active = 0',
|
||||
'SELECT trigger_type, COUNT(*) as count FROM workflows GROUP BY trigger_type',
|
||||
'SELECT complexity, COUNT(*) as count FROM workflows GROUP BY complexity',
|
||||
'SELECT SUM(node_count) as total_nodes FROM workflows',
|
||||
'SELECT analyzed_at FROM workflows ORDER BY analyzed_at DESC LIMIT 1'
|
||||
];
|
||||
|
||||
Promise.all(queries.map(sql =>
|
||||
new Promise((resolve, reject) => {
|
||||
this.db.all(sql, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows);
|
||||
});
|
||||
})
|
||||
)).then(results => {
|
||||
const [total, active, inactive, triggers, complexity, nodes, lastIndexed] = results;
|
||||
|
||||
const triggersMap = {};
|
||||
triggers.forEach(row => {
|
||||
triggersMap[row.trigger_type] = row.count;
|
||||
});
|
||||
|
||||
const complexityMap = {};
|
||||
complexity.forEach(row => {
|
||||
complexityMap[row.complexity] = row.count;
|
||||
});
|
||||
|
||||
// Count unique integrations
|
||||
this.db.all('SELECT integrations FROM workflows', (err, rows) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const allIntegrations = new Set();
|
||||
rows.forEach(row => {
|
||||
try {
|
||||
const integrations = JSON.parse(row.integrations || '[]');
|
||||
integrations.forEach(integration => allIntegrations.add(integration));
|
||||
} catch (e) {
|
||||
// Ignore parse errors
|
||||
}
|
||||
});
|
||||
|
||||
resolve({
|
||||
total: total[0].total,
|
||||
active: active[0].active,
|
||||
inactive: inactive[0].inactive,
|
||||
triggers: triggersMap,
|
||||
complexity: complexityMap,
|
||||
total_nodes: nodes[0].total_nodes || 0,
|
||||
unique_integrations: allIntegrations.size,
|
||||
last_indexed: lastIndexed[0]?.analyzed_at || ''
|
||||
});
|
||||
});
|
||||
}).catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
async getWorkflowDetail(filename) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(
|
||||
'SELECT * FROM workflows WHERE filename = ?',
|
||||
[filename],
|
||||
(err, row) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse JSON fields and load raw workflow data
|
||||
const workflow = {
|
||||
...row,
|
||||
integrations: JSON.parse(row.integrations || '[]'),
|
||||
tags: JSON.parse(row.tags || '[]')
|
||||
};
|
||||
|
||||
// Load raw workflow JSON
|
||||
try {
|
||||
const workflowPath = path.join(this.workflowsDir, filename);
|
||||
const rawWorkflow = fs.readJsonSync(workflowPath);
|
||||
workflow.raw_workflow = rawWorkflow;
|
||||
} catch (error) {
|
||||
console.error(`Error loading raw workflow ${filename}:`, error.message);
|
||||
}
|
||||
|
||||
resolve(workflow);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = WorkflowDatabase;
|
||||
97
src/index-workflows.js
Normal file
97
src/index-workflows.js
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { program } = require('commander');
|
||||
const WorkflowDatabase = require('./database');
|
||||
|
||||
function printBanner() {
|
||||
console.log('📚 N8N Workflow Indexer');
|
||||
console.log('=' .repeat(30));
|
||||
}
|
||||
|
||||
async function indexWorkflows(force = false) {
|
||||
const db = new WorkflowDatabase();
|
||||
|
||||
try {
|
||||
console.log('🔄 Starting workflow indexing...');
|
||||
await db.initialize();
|
||||
|
||||
const results = await db.indexWorkflows(force);
|
||||
|
||||
console.log('✅ Indexing completed!');
|
||||
console.log(`📊 Results:`);
|
||||
console.log(` • Processed: ${results.processed}`);
|
||||
console.log(` • Skipped: ${results.skipped}`);
|
||||
console.log(` • Errors: ${results.errors}`);
|
||||
console.log(` • Total files: ${results.total}`);
|
||||
|
||||
// Show final stats
|
||||
const stats = await db.getStats();
|
||||
console.log(`\n📈 Database Statistics:`);
|
||||
console.log(` • Total workflows: ${stats.total}`);
|
||||
console.log(` • Active workflows: ${stats.active}`);
|
||||
console.log(` • Unique integrations: ${stats.unique_integrations}`);
|
||||
console.log(` • Total nodes: ${stats.total_nodes}`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Indexing failed:', error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// CLI interface
|
||||
program
|
||||
.description('Index N8N workflows into the database')
|
||||
.option('-f, --force', 'Force reindexing of all workflows')
|
||||
.option('--stats', 'Show database statistics only')
|
||||
.parse();
|
||||
|
||||
const options = program.opts();
|
||||
|
||||
async function main() {
|
||||
printBanner();
|
||||
|
||||
const db = new WorkflowDatabase();
|
||||
|
||||
if (options.stats) {
|
||||
try {
|
||||
await db.initialize();
|
||||
const stats = await db.getStats();
|
||||
console.log('📊 Database Statistics:');
|
||||
console.log(` • Total workflows: ${stats.total}`);
|
||||
console.log(` • Active workflows: ${stats.active}`);
|
||||
console.log(` • Inactive workflows: ${stats.inactive}`);
|
||||
console.log(` • Unique integrations: ${stats.unique_integrations}`);
|
||||
console.log(` • Total nodes: ${stats.total_nodes}`);
|
||||
console.log(` • Last indexed: ${stats.last_indexed}`);
|
||||
|
||||
if (stats.triggers) {
|
||||
console.log(` • Trigger types:`);
|
||||
Object.entries(stats.triggers).forEach(([type, count]) => {
|
||||
console.log(` - ${type}: ${count}`);
|
||||
});
|
||||
}
|
||||
|
||||
if (stats.complexity) {
|
||||
console.log(` • Complexity distribution:`);
|
||||
Object.entries(stats.complexity).forEach(([level, count]) => {
|
||||
console.log(` - ${level}: ${count}`);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Error fetching stats:', error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
} else {
|
||||
await indexWorkflows(options.force);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = { indexWorkflows };
|
||||
45
src/init-db.js
Normal file
45
src/init-db.js
Normal file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const WorkflowDatabase = require('./database');
|
||||
|
||||
async function initializeDatabase() {
|
||||
console.log('🔄 Initializing N8N Workflow Database...');
|
||||
|
||||
try {
|
||||
// Ensure required directories exist
|
||||
await fs.ensureDir('database');
|
||||
await fs.ensureDir('workflows');
|
||||
await fs.ensureDir('static');
|
||||
|
||||
console.log('✅ Directories created/verified');
|
||||
|
||||
// Initialize database
|
||||
const db = new WorkflowDatabase();
|
||||
await db.initialize();
|
||||
|
||||
// Get stats to verify database works
|
||||
const stats = await db.getStats();
|
||||
console.log('✅ Database initialized successfully');
|
||||
console.log(`📊 Current stats: ${stats.total} workflows`);
|
||||
|
||||
db.close();
|
||||
|
||||
console.log('\n🎉 Initialization complete!');
|
||||
console.log('Next steps:');
|
||||
console.log('1. Place your workflow JSON files in the "workflows" directory');
|
||||
console.log('2. Run "npm run index" to index your workflows');
|
||||
console.log('3. Run "npm start" to start the server');
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Initialization failed:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
initializeDatabase();
|
||||
}
|
||||
|
||||
module.exports = { initializeDatabase };
|
||||
369
src/server.js
Normal file
369
src/server.js
Normal file
@@ -0,0 +1,369 @@
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const compression = require('compression');
|
||||
const helmet = require('helmet');
|
||||
const rateLimit = require('express-rate-limit');
|
||||
const path = require('path');
|
||||
const fs = require('fs-extra');
|
||||
const { program } = require('commander');
|
||||
|
||||
const WorkflowDatabase = require('./database');
|
||||
|
||||
// Initialize Express app
|
||||
const app = express();
|
||||
const db = new WorkflowDatabase();
|
||||
|
||||
// Security middleware
|
||||
app.use(helmet({
|
||||
contentSecurityPolicy: {
|
||||
directives: {
|
||||
defaultSrc: ["'self'"],
|
||||
styleSrc: ["'self'", "'unsafe-inline'", "https://cdn.jsdelivr.net"],
|
||||
scriptSrc: ["'self'", "'unsafe-inline'", "https://cdn.jsdelivr.net"],
|
||||
imgSrc: ["'self'", "data:", "https:"],
|
||||
connectSrc: ["'self'"],
|
||||
fontSrc: ["'self'", "https://fonts.gstatic.com"],
|
||||
objectSrc: ["'none'"],
|
||||
mediaSrc: ["'self'"],
|
||||
frameSrc: ["'none'"],
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
// Rate limiting
|
||||
const limiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 1000, // limit each IP to 1000 requests per windowMs
|
||||
message: 'Too many requests from this IP, please try again later.'
|
||||
});
|
||||
app.use('/api/', limiter);
|
||||
|
||||
// Middleware
|
||||
app.use(compression());
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({ extended: true }));
|
||||
|
||||
// Serve static files
|
||||
app.use(express.static(path.join(__dirname, '../static')));
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy', message: 'N8N Workflow API is running' });
|
||||
});
|
||||
|
||||
// Main page
|
||||
app.get('/', (req, res) => {
|
||||
const staticPath = path.join(__dirname, '../static/index.html');
|
||||
|
||||
if (fs.existsSync(staticPath)) {
|
||||
res.sendFile(staticPath);
|
||||
} else {
|
||||
res.status(404).send(`
|
||||
<html><body>
|
||||
<h1>Setup Required</h1>
|
||||
<p>Static files not found. Please ensure the static directory exists with index.html</p>
|
||||
<p>Current directory: ${process.cwd()}</p>
|
||||
</body></html>
|
||||
`);
|
||||
}
|
||||
});
|
||||
|
||||
// API Routes
|
||||
|
||||
// Get workflow statistics
|
||||
app.get('/api/stats', async (req, res) => {
|
||||
try {
|
||||
const stats = await db.getStats();
|
||||
res.json(stats);
|
||||
} catch (error) {
|
||||
console.error('Error fetching stats:', error);
|
||||
res.status(500).json({ error: 'Error fetching stats', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Search workflows
|
||||
app.get('/api/workflows', async (req, res) => {
|
||||
try {
|
||||
const {
|
||||
q = '',
|
||||
trigger = 'all',
|
||||
complexity = 'all',
|
||||
active_only = false,
|
||||
page = 1,
|
||||
per_page = 20
|
||||
} = req.query;
|
||||
|
||||
const pageNum = Math.max(1, parseInt(page));
|
||||
const perPage = Math.min(100, Math.max(1, parseInt(per_page)));
|
||||
const offset = (pageNum - 1) * perPage;
|
||||
const activeOnly = active_only === 'true';
|
||||
|
||||
const { workflows, total } = await db.searchWorkflows(
|
||||
q, trigger, complexity, activeOnly, perPage, offset
|
||||
);
|
||||
|
||||
const pages = Math.ceil(total / perPage);
|
||||
|
||||
res.json({
|
||||
workflows,
|
||||
total,
|
||||
page: pageNum,
|
||||
per_page: perPage,
|
||||
pages,
|
||||
query: q,
|
||||
filters: {
|
||||
trigger,
|
||||
complexity,
|
||||
active_only: activeOnly
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error searching workflows:', error);
|
||||
res.status(500).json({ error: 'Error searching workflows', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get workflow detail
|
||||
app.get('/api/workflows/:filename', async (req, res) => {
|
||||
try {
|
||||
const { filename } = req.params;
|
||||
const workflow = await db.getWorkflowDetail(filename);
|
||||
|
||||
if (!workflow) {
|
||||
return res.status(404).json({ error: 'Workflow not found' });
|
||||
}
|
||||
|
||||
res.json(workflow);
|
||||
} catch (error) {
|
||||
console.error('Error fetching workflow detail:', error);
|
||||
res.status(500).json({ error: 'Error fetching workflow detail', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Download workflow
|
||||
app.get('/api/workflows/:filename/download', async (req, res) => {
|
||||
try {
|
||||
const { filename } = req.params;
|
||||
const workflowPath = path.join('workflows', filename);
|
||||
|
||||
if (!fs.existsSync(workflowPath)) {
|
||||
return res.status(404).json({ error: 'Workflow file not found' });
|
||||
}
|
||||
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.sendFile(path.resolve(workflowPath));
|
||||
} catch (error) {
|
||||
console.error('Error downloading workflow:', error);
|
||||
res.status(500).json({ error: 'Error downloading workflow', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get workflow diagram (Mermaid)
|
||||
app.get('/api/workflows/:filename/diagram', async (req, res) => {
|
||||
try {
|
||||
const { filename } = req.params;
|
||||
const workflow = await db.getWorkflowDetail(filename);
|
||||
|
||||
if (!workflow || !workflow.raw_workflow) {
|
||||
return res.status(404).json({ error: 'Workflow not found' });
|
||||
}
|
||||
|
||||
const diagram = generateMermaidDiagram(workflow.raw_workflow.nodes, workflow.raw_workflow.connections);
|
||||
res.json({ diagram });
|
||||
} catch (error) {
|
||||
console.error('Error generating diagram:', error);
|
||||
res.status(500).json({ error: 'Error generating diagram', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Generate Mermaid diagram
|
||||
function generateMermaidDiagram(nodes, connections) {
|
||||
if (!nodes || nodes.length === 0) {
|
||||
return 'graph TD\n A[No nodes found]';
|
||||
}
|
||||
|
||||
let diagram = 'graph TD\n';
|
||||
|
||||
// Add nodes
|
||||
nodes.forEach(node => {
|
||||
const nodeId = sanitizeNodeId(node.name);
|
||||
const nodeType = node.type?.split('.').pop() || 'unknown';
|
||||
diagram += ` ${nodeId}["${node.name}\\n(${nodeType})"]\n`;
|
||||
});
|
||||
|
||||
// Add connections
|
||||
if (connections) {
|
||||
Object.entries(connections).forEach(([sourceNode, outputs]) => {
|
||||
const sourceId = sanitizeNodeId(sourceNode);
|
||||
|
||||
outputs.main?.forEach(outputConnections => {
|
||||
outputConnections.forEach(connection => {
|
||||
const targetId = sanitizeNodeId(connection.node);
|
||||
diagram += ` ${sourceId} --> ${targetId}\n`;
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return diagram;
|
||||
}
|
||||
|
||||
function sanitizeNodeId(nodeName) {
|
||||
// Convert node name to valid Mermaid ID
|
||||
return nodeName.replace(/[^a-zA-Z0-9]/g, '_').replace(/^_+|_+$/g, '');
|
||||
}
|
||||
|
||||
// Reindex workflows
|
||||
app.post('/api/reindex', async (req, res) => {
|
||||
try {
|
||||
const { force = false } = req.body;
|
||||
|
||||
// Run indexing in background
|
||||
db.indexWorkflows(force).then(results => {
|
||||
console.log('Indexing completed:', results);
|
||||
}).catch(error => {
|
||||
console.error('Indexing error:', error);
|
||||
});
|
||||
|
||||
res.json({ message: 'Indexing started in background' });
|
||||
} catch (error) {
|
||||
console.error('Error starting reindex:', error);
|
||||
res.status(500).json({ error: 'Error starting reindex', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get integrations
|
||||
app.get('/api/integrations', async (req, res) => {
|
||||
try {
|
||||
const { workflows } = await db.searchWorkflows('', 'all', 'all', false, 1000, 0);
|
||||
|
||||
const integrations = new Set();
|
||||
workflows.forEach(workflow => {
|
||||
workflow.integrations.forEach(integration => integrations.add(integration));
|
||||
});
|
||||
|
||||
res.json(Array.from(integrations).sort());
|
||||
} catch (error) {
|
||||
console.error('Error fetching integrations:', error);
|
||||
res.status(500).json({ error: 'Error fetching integrations', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get categories (based on integrations)
|
||||
app.get('/api/categories', async (req, res) => {
|
||||
try {
|
||||
const { workflows } = await db.searchWorkflows('', 'all', 'all', false, 1000, 0);
|
||||
|
||||
const categories = {
|
||||
'Communication': ['Slack', 'Discord', 'Telegram', 'Mattermost', 'Teams'],
|
||||
'CRM': ['HubSpot', 'Salesforce', 'Pipedrive', 'Copper'],
|
||||
'Data': ['GoogleSheets', 'Airtable', 'Mysql', 'Postgres'],
|
||||
'Development': ['GitHub', 'GitLab', 'Jira', 'Trello'],
|
||||
'Marketing': ['Mailchimp', 'Sendinblue', 'Typeform', 'Webflow'],
|
||||
'Storage': ['GoogleDrive', 'Dropbox', 'OneDrive', 'AWS S3'],
|
||||
'Other': []
|
||||
};
|
||||
|
||||
// Categorize workflows
|
||||
const categorizedWorkflows = {};
|
||||
Object.keys(categories).forEach(category => {
|
||||
categorizedWorkflows[category] = [];
|
||||
});
|
||||
|
||||
workflows.forEach(workflow => {
|
||||
let categorized = false;
|
||||
|
||||
// Check each integration against categories
|
||||
workflow.integrations.forEach(integration => {
|
||||
Object.entries(categories).forEach(([category, services]) => {
|
||||
if (services.some(service =>
|
||||
integration.toLowerCase().includes(service.toLowerCase())
|
||||
)) {
|
||||
categorizedWorkflows[category].push(workflow);
|
||||
categorized = true;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// If not categorized, add to Other
|
||||
if (!categorized) {
|
||||
categorizedWorkflows['Other'].push(workflow);
|
||||
}
|
||||
});
|
||||
|
||||
res.json(categorizedWorkflows);
|
||||
} catch (error) {
|
||||
console.error('Error fetching categories:', error);
|
||||
res.status(500).json({ error: 'Error fetching categories', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Error handler
|
||||
app.use((error, req, res, next) => {
|
||||
console.error('Unhandled error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Internal server error',
|
||||
details: process.env.NODE_ENV === 'development' ? error.message : undefined
|
||||
});
|
||||
});
|
||||
|
||||
// 404 handler
|
||||
app.use((req, res) => {
|
||||
res.status(404).json({ error: 'Not found' });
|
||||
});
|
||||
|
||||
// Start server
|
||||
function startServer(port = 8000, host = '127.0.0.1') {
|
||||
const server = app.listen(port, host, () => {
|
||||
console.log('🚀 N8N Workflow Documentation Server');
|
||||
console.log('=' .repeat(50));
|
||||
console.log(`🌐 Server running at http://${host}:${port}`);
|
||||
console.log(`📊 API Documentation: http://${host}:${port}/api/stats`);
|
||||
console.log(`🔍 Workflow Search: http://${host}:${port}/api/workflows`);
|
||||
console.log();
|
||||
console.log('Press Ctrl+C to stop the server');
|
||||
console.log('-'.repeat(50));
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
process.on('SIGINT', () => {
|
||||
console.log('\n👋 Shutting down server...');
|
||||
server.close(() => {
|
||||
db.close();
|
||||
console.log('✅ Server stopped');
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// CLI interface
|
||||
if (require.main === module) {
|
||||
program
|
||||
.option('-p, --port <port>', 'Port to run server on', '8000')
|
||||
.option('-h, --host <host>', 'Host to bind to', '127.0.0.1')
|
||||
.option('--dev', 'Enable development mode')
|
||||
.parse();
|
||||
|
||||
const options = program.opts();
|
||||
const port = parseInt(options.port);
|
||||
const host = options.host;
|
||||
|
||||
// Check if database needs initialization
|
||||
db.initialize().then(() => {
|
||||
return db.getStats();
|
||||
}).then(stats => {
|
||||
if (stats.total === 0) {
|
||||
console.log('⚠️ Warning: No workflows found. Run "npm run index" to index workflows.');
|
||||
} else {
|
||||
console.log(`✅ Database ready: ${stats.total} workflows indexed`);
|
||||
}
|
||||
startServer(port, host);
|
||||
}).catch(error => {
|
||||
console.error('❌ Database connection failed:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = app;
|
||||
Reference in New Issue
Block a user