#!/usr/bin/env node /** * Advanced Flatfile Docs Search Tool * * This tool provides advanced search capabilities for Flatfile documentation, * including web scraping, local file search, and AI-powered content analysis. */ const https = require('https'); const fs = require('fs'); const path = require('path'); class AdvancedFlatfileDocsSearch { constructor() { this.baseUrl = 'https://flatfile.com'; this.docsPath = '/docs'; this.localDocsPath = path.join(__dirname, '..', '..', '..'); this.searchCache = new Map(); this.cacheTimeout = 5 * 60 * 1000; // 5 minutes } /** * Main search function that tries multiple approaches * @param {string} query - The search query * @param {Object} options - Search options * @returns {Promise} Search results */ async search(query, options = {}) { const { limit = 10, includeContent = false, searchType = 'hybrid', // 'web', 'local', 'hybrid' category = null, // Filter by category sortBy = 'relevance' // 'relevance', 'date', 'title' } = options; try { let results = []; // Check cache first const cacheKey = `${query}-${JSON.stringify(options)}`; if (this.searchCache.has(cacheKey)) { const cached = this.searchCache.get(cacheKey); if (Date.now() - cached.timestamp < this.cacheTimeout) { return cached.results; } } switch (searchType) { case 'web': results = await this.searchWeb(query, limit); break; case 'local': results = await this.searchLocal(query, { limit, includeContent, category }); break; case 'hybrid': default: // Try web first, fall back to local results = await this.searchWeb(query, limit); if (!results || results.length === 0) { results = await this.searchLocal(query, { limit, includeContent, category }); } break; } // Apply additional filtering and sorting if (category) { results = this.filterByCategory(results, category); } results = this.sortResults(results, sortBy); // Cache results this.searchCache.set(cacheKey, { results: results.slice(0, limit), timestamp: Date.now() }); return results.slice(0, limit); } catch (error) { console.error('Search error:', error.message); return []; } } /** * Search via web scraping the docs site * @param {string} query - The search query * @param {number} limit - Number of results to return * @returns {Promise} Search results */ async searchWeb(query, limit) { return new Promise((resolve) => { // Try to access the search functionality via different approaches const searchUrl = `${this.baseUrl}/docs`; const req = https.get(searchUrl, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { // Parse HTML to find search functionality const searchResults = this.parseSearchFromHTML(data, query); resolve(searchResults); }); }); req.on('error', () => { resolve(null); }); req.setTimeout(10000, () => { req.destroy(); resolve(null); }); }); } /** * Parse HTML to extract search functionality * @param {string} html - HTML content * @param {string} query - Search query * @returns {Array} Search results */ parseSearchFromHTML(html, query) { // This is a simplified approach - in a real implementation, // you might use a library like cheerio to parse HTML const results = []; // Look for search API endpoints in the HTML const searchApiMatch = html.match(/search[^"]*api[^"]*["']([^"']+)["']/i); if (searchApiMatch) { // Found a search API endpoint - could make another request here console.log('Found potential search API:', searchApiMatch[1]); } // Look for search functionality scripts const searchScriptMatch = html.match(/search[^}]*{[^}]*}/gi); if (searchScriptMatch) { console.log('Found search scripts'); } return results; } /** * Enhanced local search with advanced features * @param {string} query - The search query * @param {Object} options - Search options * @returns {Promise} Search results */ async searchLocal(query, options = {}) { const { limit = 10, includeContent = false, category = null } = options; const results = []; const queryLower = query.toLowerCase(); const queryWords = queryLower.split(/\s+/).filter(word => word.length > 2); try { // Read docs.json to get all pages const docsJson = JSON.parse(fs.readFileSync(path.join(this.localDocsPath, 'docs.json'), 'utf8')); const pages = this.extractPagesFromNavigation(docsJson.navigation); // Search through each page for (const page of pages) { const pageCategory = this.getCategoryFromPath(page); // Skip if category filter doesn't match if (category && pageCategory !== category) { continue; } const filePath = path.join(this.localDocsPath, `${page}.mdx`); // Try both .mdx and .md extensions let content = ''; if (fs.existsSync(filePath)) { content = fs.readFileSync(filePath, 'utf8'); } else { const mdPath = path.join(this.localDocsPath, `${page}.md`); if (fs.existsSync(mdPath)) { content = fs.readFileSync(mdPath, 'utf8'); } } if (content) { const relevance = this.calculateAdvancedRelevance(content, queryLower, queryWords); if (relevance > 0) { const title = this.extractTitle(content, page); const description = this.extractDescription(content); const snippets = this.extractSnippets(content, queryLower, 3); const headings = this.extractHeadings(content); const result = { title, description, url: `/docs/${page}`, path: page, category: pageCategory, snippets, headings, relevance, wordCount: content.split(/\s+/).length, lastModified: this.getFileModifiedTime(filePath) }; if (includeContent) { result.content = content; } results.push(result); } } } return results; } catch (error) { console.error('Local search error:', error.message); return []; } } /** * Calculate advanced relevance score * @param {string} content - File content * @param {string} query - Search query (lowercase) * @param {Array} queryWords - Query words array * @returns {number} Relevance score */ calculateAdvancedRelevance(content, query, queryWords) { const contentLower = content.toLowerCase(); let score = 0; // Exact phrase matches (high weight) const exactMatches = (contentLower.match(new RegExp(query, 'g')) || []).length; score += exactMatches * 10; // Individual word matches queryWords.forEach(word => { const matches = (contentLower.match(new RegExp(`\\b${word}\\b`, 'g')) || []).length; score += matches * 2; // Partial matches const partialMatches = (contentLower.match(new RegExp(word, 'g')) || []).length; score += (partialMatches - matches) * 0.5; }); // Title matches (higher weight) const titleMatch = this.extractTitle(content, '').toLowerCase(); if (titleMatch.includes(query)) { score += 20; } queryWords.forEach(word => { if (titleMatch.includes(word)) { score += 5; } }); // Heading matches const headings = this.extractHeadings(content); headings.forEach(heading => { const headingLower = heading.toLowerCase(); if (headingLower.includes(query)) { score += 5; } queryWords.forEach(word => { if (headingLower.includes(word)) { score += 2; } }); }); return score; } /** * Extract description from content * @param {string} content - File content * @returns {string} Description */ extractDescription(content) { // Try to extract from frontmatter const frontmatterMatch = content.match(/^---\s*\n(.*?)\n---/s); if (frontmatterMatch) { const descMatch = frontmatterMatch[1].match(/description:\s*["'](.+?)["']/); if (descMatch) { return descMatch[1]; } } // Try to extract from first paragraph const paragraphMatch = content.match(/^(?:#{1,6}\s+.+\n\n)?([^#\n]+)/m); if (paragraphMatch) { return paragraphMatch[1].trim().substring(0, 200) + '...'; } return ''; } /** * Extract headings from content * @param {string} content - File content * @returns {Array} Array of headings */ extractHeadings(content) { const headings = []; const headingMatches = content.match(/^#{1,6}\s+(.+)/gm); if (headingMatches) { headingMatches.forEach(match => { const heading = match.replace(/^#{1,6}\s+/, '').trim(); headings.push(heading); }); } return headings; } /** * Get category from file path * @param {string} filePath - File path * @returns {string} Category */ getCategoryFromPath(filePath) { const parts = filePath.split('/'); return parts[0] || 'general'; } /** * Get file modification time * @param {string} filePath - File path * @returns {Date} Modification time */ getFileModifiedTime(filePath) { try { const stats = fs.statSync(filePath); return stats.mtime; } catch (e) { return new Date(0); } } /** * Filter results by category * @param {Array} results - Search results * @param {string} category - Category to filter by * @returns {Array} Filtered results */ filterByCategory(results, category) { return results.filter(result => result.category === category); } /** * Sort results by different criteria * @param {Array} results - Search results * @param {string} sortBy - Sort criteria * @returns {Array} Sorted results */ sortResults(results, sortBy) { switch (sortBy) { case 'date': return results.sort((a, b) => new Date(b.lastModified) - new Date(a.lastModified)); case 'title': return results.sort((a, b) => a.title.localeCompare(b.title)); case 'relevance': default: return results.sort((a, b) => b.relevance - a.relevance); } } /** * Extract all pages from navigation structure * @param {Object} navigation - Navigation object from docs.json * @returns {Array} Array of page paths */ extractPagesFromNavigation(navigation) { const pages = []; const extractPages = (items) => { if (Array.isArray(items)) { items.forEach(item => { if (typeof item === 'string') { pages.push(item); } else if (item.pages) { extractPages(item.pages); } }); } }; navigation.groups.forEach(group => { extractPages(group.pages); }); return pages; } /** * Extract title from content * @param {string} content - File content * @param {string} fallback - Fallback title * @returns {string} Extracted title */ extractTitle(content, fallback) { // Try to extract from frontmatter const frontmatterMatch = content.match(/^---\s*\n(.*?)\n---/s); if (frontmatterMatch) { const titleMatch = frontmatterMatch[1].match(/title:\s*["'](.+?)["']/); if (titleMatch) { return titleMatch[1]; } } // Try to extract from first heading const headingMatch = content.match(/^#\s+(.+)/m); if (headingMatch) { return headingMatch[1]; } // Use fallback return fallback.split('/').pop().replace(/[-_]/g, ' '); } /** * Extract relevant snippets containing the search query * @param {string} content - File content * @param {string} query - Search query (lowercase) * @param {number} maxSnippets - Maximum number of snippets * @returns {Array} Array of relevant snippets */ extractSnippets(content, query, maxSnippets) { const snippets = []; const lines = content.split('\n'); for (let i = 0; i < lines.length && snippets.length < maxSnippets; i++) { const line = lines[i]; if (line.toLowerCase().includes(query)) { // Get context around the match const start = Math.max(0, i - 1); const end = Math.min(lines.length, i + 2); const snippet = lines.slice(start, end).join('\n').trim(); if (snippet.length > 0) { snippets.push(snippet); } } } return snippets; } /** * Get available categories * @returns {Array} Array of categories */ getAvailableCategories() { try { const docsJson = JSON.parse(fs.readFileSync(path.join(this.localDocsPath, 'docs.json'), 'utf8')); return docsJson.navigation.groups.map(group => ({ name: group.group, icon: group.icon, key: group.group.toLowerCase().replace(/[^a-z0-9]/g, '-') })); } catch (error) { return []; } } /** * Format search results for display * @param {Array} results - Search results * @param {Object} options - Display options * @returns {string} Formatted results */ formatResults(results, options = {}) { const { showSnippets = true, showCategory = true, showRelevance = false, maxSnippetLength = 150 } = options; if (!results || results.length === 0) { return 'No results found.'; } let output = `Found ${results.length} result(s):\n\n`; results.forEach((result, index) => { output += `${index + 1}. **${result.title}**\n`; if (result.description) { output += ` ${result.description}\n`; } output += ` URL: https://flatfile.com${result.url}\n`; if (showCategory && result.category) { output += ` Category: ${result.category}\n`; } if (showRelevance && result.relevance) { output += ` Relevance: ${result.relevance.toFixed(2)}\n`; } if (showSnippets && result.snippets && result.snippets.length > 0) { output += ` Preview: ${result.snippets[0].substring(0, maxSnippetLength)}...\n`; } output += '\n'; }); return output; } } // CLI interface async function main() { const args = process.argv.slice(2); if (args.length === 0) { console.log('Usage: node advanced-search.js [options]'); console.log(''); console.log('Options:'); console.log(' --limit Number of results (default: 10)'); console.log(' --category Filter by category'); console.log(' --sort Sort by: relevance, date, title (default: relevance)'); console.log(' --type Search type: web, local, hybrid (default: hybrid)'); console.log(' --include-content Include full content in results'); console.log(' --show-categories Show available categories'); console.log(''); console.log('Examples:'); console.log(' node advanced-search.js "authentication" --limit 5'); console.log(' node advanced-search.js "webhook" --category guides --sort date'); console.log(' node advanced-search.js --show-categories'); process.exit(1); } const search = new AdvancedFlatfileDocsSearch(); // Handle special commands if (args.includes('--show-categories')) { const categories = search.getAvailableCategories(); console.log('Available categories:'); categories.forEach(cat => { console.log(` ${cat.key}: ${cat.name} (${cat.icon})`); }); return; } const query = args[0]; const options = {}; // Parse options for (let i = 1; i < args.length; i++) { switch (args[i]) { case '--limit': options.limit = parseInt(args[i + 1]) || 10; i++; break; case '--category': options.category = args[i + 1]; i++; break; case '--sort': options.sortBy = args[i + 1]; i++; break; case '--type': options.searchType = args[i + 1]; i++; break; case '--include-content': options.includeContent = true; break; } } console.log(`Searching for: "${query}"`); if (options.category) { console.log(`Category: ${options.category}`); } console.log('---'); const results = await search.search(query, options); console.log(search.formatResults(results, { showCategory: true, showRelevance: true })); } // Export for use as module module.exports = AdvancedFlatfileDocsSearch; // Run as CLI if called directly if (require.main === module) { main().catch(console.error); }

Getting Started

Core Concepts

Coding Tutorial

Plugins

Advanced Guides

Embedding Flatfile

Reference

Documentation Index

Subscribe to updates via RSS