/* Custom styles for Flatfile docs */

/* Monospace font for code */
@font-face {
  font-family: 'Flatfile Diatype Mono';
  src: url('https://flatfile.com/fonts/FlatfileDiatypeMono-Regular.woff2') format('woff2');
  font-weight: 400;
  font-style: normal;
  font-display: swap;
}

/* Apply monospace font to code elements */
code, pre, .code {
  font-family: 'Flatfile Diatype Mono', 'SF Mono', Monaco, Consolas, 'Courier New', monospace !important;
}

em {
  font-style: oblique 12deg;
}

#!/usr/bin/env node

/**
 * Advanced Flatfile Docs Search Tool
 * 
 * This tool provides advanced search capabilities for Flatfile documentation,
 * including web scraping, local file search, and AI-powered content analysis.
 */

const https = require('https');
const fs = require('fs');
const path = require('path');

class AdvancedFlatfileDocsSearch {
  constructor() {
    this.baseUrl = 'https://flatfile.com';
    this.docsPath = '/docs';
    this.localDocsPath = path.join(__dirname, '..', '..', '..');
    this.searchCache = new Map();
    this.cacheTimeout = 5 * 60 * 1000; // 5 minutes
  }

  /**
   * Main search function that tries multiple approaches
   * @param {string} query - The search query
   * @param {Object} options - Search options
   * @returns {Promise<Array>} Search results
   */
  async search(query, options = {}) {
    const {
      limit = 10,
      includeContent = false,
      searchType = 'hybrid', // 'web', 'local', 'hybrid'
      category = null, // Filter by category
      sortBy = 'relevance' // 'relevance', 'date', 'title'
    } = options;

    try {
      let results = [];

      // Check cache first
      const cacheKey = `${query}-${JSON.stringify(options)}`;
      if (this.searchCache.has(cacheKey)) {
        const cached = this.searchCache.get(cacheKey);
        if (Date.now() - cached.timestamp < this.cacheTimeout) {
          return cached.results;
        }
      }

      switch (searchType) {
        case 'web':
          results = await this.searchWeb(query, limit);
          break;
        case 'local':
          results = await this.searchLocal(query, { limit, includeContent, category });
          break;
        case 'hybrid':
        default:
          // Try web first, fall back to local
          results = await this.searchWeb(query, limit);
          if (!results || results.length === 0) {
            results = await this.searchLocal(query, { limit, includeContent, category });
          }
          break;
      }

      // Apply additional filtering and sorting
      if (category) {
        results = this.filterByCategory(results, category);
      }

      results = this.sortResults(results, sortBy);

      // Cache results
      this.searchCache.set(cacheKey, {
        results: results.slice(0, limit),
        timestamp: Date.now()
      });

      return results.slice(0, limit);
    } catch (error) {
      console.error('Search error:', error.message);
      return [];
    }
  }

  /**
   * Search via web scraping the docs site
   * @param {string} query - The search query
   * @param {number} limit - Number of results to return
   * @returns {Promise<Array>} Search results
   */
  async searchWeb(query, limit) {
    return new Promise((resolve) => {
      // Try to access the search functionality via different approaches
      const searchUrl = `${this.baseUrl}/docs`;
      
      const req = https.get(searchUrl, (res) => {
        let data = '';
        
        res.on('data', (chunk) => {
          data += chunk;
        });
        
        res.on('end', () => {
          // Parse HTML to find search functionality
          const searchResults = this.parseSearchFromHTML(data, query);
          resolve(searchResults);
        });
      });

      req.on('error', () => {
        resolve(null);
      });

      req.setTimeout(10000, () => {
        req.destroy();
        resolve(null);
      });
    });
  }

  /**
   * Parse HTML to extract search functionality
   * @param {string} html - HTML content
   * @param {string} query - Search query
   * @returns {Array} Search results
   */
  parseSearchFromHTML(html, query) {
    // This is a simplified approach - in a real implementation,
    // you might use a library like cheerio to parse HTML
    const results = [];
    
    // Look for search API endpoints in the HTML
    const searchApiMatch = html.match(/search[^"]*api[^"]*["']([^"']+)["']/i);
    if (searchApiMatch) {
      // Found a search API endpoint - could make another request here
      console.log('Found potential search API:', searchApiMatch[1]);
    }
    
    // Look for search functionality scripts
    const searchScriptMatch = html.match(/search[^}]*{[^}]*}/gi);
    if (searchScriptMatch) {
      console.log('Found search scripts');
    }
    
    return results;
  }

  /**
   * Enhanced local search with advanced features
   * @param {string} query - The search query
   * @param {Object} options - Search options
   * @returns {Promise<Array>} Search results
   */
  async searchLocal(query, options = {}) {
    const { limit = 10, includeContent = false, category = null } = options;
    const results = [];
    const queryLower = query.toLowerCase();
    const queryWords = queryLower.split(/\s+/).filter(word => word.length > 2);
    
    try {
      // Read docs.json to get all pages
      const docsJson = JSON.parse(fs.readFileSync(path.join(this.localDocsPath, 'docs.json'), 'utf8'));
      const pages = this.extractPagesFromNavigation(docsJson.navigation);
      
      // Search through each page
      for (const page of pages) {
        const pageCategory = this.getCategoryFromPath(page);
        
        // Skip if category filter doesn't match
        if (category && pageCategory !== category) {
          continue;
        }
        
        const filePath = path.join(this.localDocsPath, `${page}.mdx`);
        
        // Try both .mdx and .md extensions
        let content = '';
        if (fs.existsSync(filePath)) {
          content = fs.readFileSync(filePath, 'utf8');
        } else {
          const mdPath = path.join(this.localDocsPath, `${page}.md`);
          if (fs.existsSync(mdPath)) {
            content = fs.readFileSync(mdPath, 'utf8');
          }
        }
        
        if (content) {
          const relevance = this.calculateAdvancedRelevance(content, queryLower, queryWords);
          
          if (relevance > 0) {
            const title = this.extractTitle(content, page);
            const description = this.extractDescription(content);
            const snippets = this.extractSnippets(content, queryLower, 3);
            const headings = this.extractHeadings(content);
            
            const result = {
              title,
              description,
              url: `/docs/${page}`,
              path: page,
              category: pageCategory,
              snippets,
              headings,
              relevance,
              wordCount: content.split(/\s+/).length,
              lastModified: this.getFileModifiedTime(filePath)
            };
            
            if (includeContent) {
              result.content = content;
            }
            
            results.push(result);
          }
        }
      }
      
      return results;
      
    } catch (error) {
      console.error('Local search error:', error.message);
      return [];
    }
  }

  /**
   * Calculate advanced relevance score
   * @param {string} content - File content
   * @param {string} query - Search query (lowercase)
   * @param {Array} queryWords - Query words array
   * @returns {number} Relevance score
   */
  calculateAdvancedRelevance(content, query, queryWords) {
    const contentLower = content.toLowerCase();
    let score = 0;
    
    // Exact phrase matches (high weight)
    const exactMatches = (contentLower.match(new RegExp(query, 'g')) || []).length;
    score += exactMatches * 10;
    
    // Individual word matches
    queryWords.forEach(word => {
      const matches = (contentLower.match(new RegExp(`\\b${word}\\b`, 'g')) || []).length;
      score += matches * 2;
      
      // Partial matches
      const partialMatches = (contentLower.match(new RegExp(word, 'g')) || []).length;
      score += (partialMatches - matches) * 0.5;
    });
    
    // Title matches (higher weight)
    const titleMatch = this.extractTitle(content, '').toLowerCase();
    if (titleMatch.includes(query)) {
      score += 20;
    }
    
    queryWords.forEach(word => {
      if (titleMatch.includes(word)) {
        score += 5;
      }
    });
    
    // Heading matches
    const headings = this.extractHeadings(content);
    headings.forEach(heading => {
      const headingLower = heading.toLowerCase();
      if (headingLower.includes(query)) {
        score += 5;
      }
      queryWords.forEach(word => {
        if (headingLower.includes(word)) {
          score += 2;
        }
      });
    });
    
    return score;
  }

  /**
   * Extract description from content
   * @param {string} content - File content
   * @returns {string} Description
   */
  extractDescription(content) {
    // Try to extract from frontmatter
    const frontmatterMatch = content.match(/^---\s*\n(.*?)\n---/s);
    if (frontmatterMatch) {
      const descMatch = frontmatterMatch[1].match(/description:\s*["'](.+?)["']/);
      if (descMatch) {
        return descMatch[1];
      }
    }
    
    // Try to extract from first paragraph
    const paragraphMatch = content.match(/^(?:#{1,6}\s+.+\n\n)?([^#\n]+)/m);
    if (paragraphMatch) {
      return paragraphMatch[1].trim().substring(0, 200) + '...';
    }
    
    return '';
  }

  /**
   * Extract headings from content
   * @param {string} content - File content
   * @returns {Array} Array of headings
   */
  extractHeadings(content) {
    const headings = [];
    const headingMatches = content.match(/^#{1,6}\s+(.+)/gm);
    
    if (headingMatches) {
      headingMatches.forEach(match => {
        const heading = match.replace(/^#{1,6}\s+/, '').trim();
        headings.push(heading);
      });
    }
    
    return headings;
  }

  /**
   * Get category from file path
   * @param {string} filePath - File path
   * @returns {string} Category
   */
  getCategoryFromPath(filePath) {
    const parts = filePath.split('/');
    return parts[0] || 'general';
  }

  /**
   * Get file modification time
   * @param {string} filePath - File path
   * @returns {Date} Modification time
   */
  getFileModifiedTime(filePath) {
    try {
      const stats = fs.statSync(filePath);
      return stats.mtime;
    } catch (e) {
      return new Date(0);
    }
  }

  /**
   * Filter results by category
   * @param {Array} results - Search results
   * @param {string} category - Category to filter by
   * @returns {Array} Filtered results
   */
  filterByCategory(results, category) {
    return results.filter(result => result.category === category);
  }

  /**
   * Sort results by different criteria
   * @param {Array} results - Search results
   * @param {string} sortBy - Sort criteria
   * @returns {Array} Sorted results
   */
  sortResults(results, sortBy) {
    switch (sortBy) {
      case 'date':
        return results.sort((a, b) => new Date(b.lastModified) - new Date(a.lastModified));
      case 'title':
        return results.sort((a, b) => a.title.localeCompare(b.title));
      case 'relevance':
      default:
        return results.sort((a, b) => b.relevance - a.relevance);
    }
  }

  /**
   * Extract all pages from navigation structure
   * @param {Object} navigation - Navigation object from docs.json
   * @returns {Array} Array of page paths
   */
  extractPagesFromNavigation(navigation) {
    const pages = [];
    
    const extractPages = (items) => {
      if (Array.isArray(items)) {
        items.forEach(item => {
          if (typeof item === 'string') {
            pages.push(item);
          } else if (item.pages) {
            extractPages(item.pages);
          }
        });
      }
    };
    
    navigation.groups.forEach(group => {
      extractPages(group.pages);
    });
    
    return pages;
  }

  /**
   * Extract title from content
   * @param {string} content - File content
   * @param {string} fallback - Fallback title
   * @returns {string} Extracted title
   */
  extractTitle(content, fallback) {
    // Try to extract from frontmatter
    const frontmatterMatch = content.match(/^---\s*\n(.*?)\n---/s);
    if (frontmatterMatch) {
      const titleMatch = frontmatterMatch[1].match(/title:\s*["'](.+?)["']/);
      if (titleMatch) {
        return titleMatch[1];
      }
    }
    
    // Try to extract from first heading
    const headingMatch = content.match(/^#\s+(.+)/m);
    if (headingMatch) {
      return headingMatch[1];
    }
    
    // Use fallback
    return fallback.split('/').pop().replace(/[-_]/g, ' ');
  }

  /**
   * Extract relevant snippets containing the search query
   * @param {string} content - File content
   * @param {string} query - Search query (lowercase)
   * @param {number} maxSnippets - Maximum number of snippets
   * @returns {Array} Array of relevant snippets
   */
  extractSnippets(content, query, maxSnippets) {
    const snippets = [];
    const lines = content.split('\n');
    
    for (let i = 0; i < lines.length && snippets.length < maxSnippets; i++) {
      const line = lines[i];
      if (line.toLowerCase().includes(query)) {
        // Get context around the match
        const start = Math.max(0, i - 1);
        const end = Math.min(lines.length, i + 2);
        const snippet = lines.slice(start, end).join('\n').trim();
        
        if (snippet.length > 0) {
          snippets.push(snippet);
        }
      }
    }
    
    return snippets;
  }

  /**
   * Get available categories
   * @returns {Array} Array of categories
   */
  getAvailableCategories() {
    try {
      const docsJson = JSON.parse(fs.readFileSync(path.join(this.localDocsPath, 'docs.json'), 'utf8'));
      return docsJson.navigation.groups.map(group => ({
        name: group.group,
        icon: group.icon,
        key: group.group.toLowerCase().replace(/[^a-z0-9]/g, '-')
      }));
    } catch (error) {
      return [];
    }
  }

  /**
   * Format search results for display
   * @param {Array} results - Search results
   * @param {Object} options - Display options
   * @returns {string} Formatted results
   */
  formatResults(results, options = {}) {
    const { 
      showSnippets = true, 
      showCategory = true, 
      showRelevance = false,
      maxSnippetLength = 150 
    } = options;

    if (!results || results.length === 0) {
      return 'No results found.';
    }

    let output = `Found ${results.length} result(s):\n\n`;
    
    results.forEach((result, index) => {
      output += `${index + 1}. **${result.title}**\n`;
      
      if (result.description) {
        output += `   ${result.description}\n`;
      }
      
      output += `   URL: https://flatfile.com${result.url}\n`;
      
      if (showCategory && result.category) {
        output += `   Category: ${result.category}\n`;
      }
      
      if (showRelevance && result.relevance) {
        output += `   Relevance: ${result.relevance.toFixed(2)}\n`;
      }
      
      if (showSnippets && result.snippets && result.snippets.length > 0) {
        output += `   Preview: ${result.snippets[0].substring(0, maxSnippetLength)}...\n`;
      }
      
      output += '\n';
    });
    
    return output;
  }
}

// CLI interface
async function main() {
  const args = process.argv.slice(2);
  
  if (args.length === 0) {
    console.log('Usage: node advanced-search.js <search-query> [options]');
    console.log('');
    console.log('Options:');
    console.log('  --limit <n>           Number of results (default: 10)');
    console.log('  --category <cat>      Filter by category');
    console.log('  --sort <method>       Sort by: relevance, date, title (default: relevance)');
    console.log('  --type <type>         Search type: web, local, hybrid (default: hybrid)');
    console.log('  --include-content     Include full content in results');
    console.log('  --show-categories     Show available categories');
    console.log('');
    console.log('Examples:');
    console.log('  node advanced-search.js "authentication" --limit 5');
    console.log('  node advanced-search.js "webhook" --category guides --sort date');
    console.log('  node advanced-search.js --show-categories');
    process.exit(1);
  }
  
  const search = new AdvancedFlatfileDocsSearch();
  
  // Handle special commands
  if (args.includes('--show-categories')) {
    const categories = search.getAvailableCategories();
    console.log('Available categories:');
    categories.forEach(cat => {
      console.log(`  ${cat.key}: ${cat.name} (${cat.icon})`);
    });
    return;
  }
  
  const query = args[0];
  const options = {};
  
  // Parse options
  for (let i = 1; i < args.length; i++) {
    switch (args[i]) {
      case '--limit':
        options.limit = parseInt(args[i + 1]) || 10;
        i++;
        break;
      case '--category':
        options.category = args[i + 1];
        i++;
        break;
      case '--sort':
        options.sortBy = args[i + 1];
        i++;
        break;
      case '--type':
        options.searchType = args[i + 1];
        i++;
        break;
      case '--include-content':
        options.includeContent = true;
        break;
    }
  }
  
  console.log(`Searching for: "${query}"`);
  if (options.category) {
    console.log(`Category: ${options.category}`);
  }
  console.log('---');
  
  const results = await search.search(query, options);
  console.log(search.formatResults(results, { showCategory: true, showRelevance: true }));
}

// Export for use as module
module.exports = AdvancedFlatfileDocsSearch;

// Run as CLI if called directly
if (require.main === module) {
  main().catch(console.error);
}


#!/usr/bin/env node

/**
 * Flatfile Docs Search Tool
 * 
 * This tool interacts with the Flatfile documentation search functionality
 * to help find relevant documentation content programmatically.
 */

const https = require('https');
const fs = require('fs');
const path = require('path');

class FlatfileDocsSearch {
  constructor() {
    this.baseUrl = 'https://flatfile.com';
    this.docsPath = '/docs';
    this.searchEndpoint = '/api/search'; // Common search endpoint pattern
    this.localDocsPath = path.join(__dirname, '..', '..', '..');
  }

  /**
   * Search the Flatfile docs
   * @param {string} query - The search query
   * @param {number} limit - Number of results to return
   * @returns {Promise<Array>} Search results
   */
  async search(query, limit = 10) {
    try {
      // First, try to search using the web API
      const webResults = await this.searchWeb(query, limit);
      
      // If web search fails, fall back to local search
      if (!webResults || webResults.length === 0) {
        console.log('Web search failed, falling back to local search...');
        return await this.searchLocal(query, limit);
      }
      
      return webResults;
    } catch (error) {
      console.error('Search error:', error.message);
      // Fall back to local search
      return await this.searchLocal(query, limit);
    }
  }

  /**
   * Search via web API (if available)
   * @param {string} query - The search query
   * @param {number} limit - Number of results to return
   * @returns {Promise<Array>} Search results
   */
  async searchWeb(query, limit) {
    return new Promise((resolve, reject) => {
      // Try common search API patterns
      const searchUrls = [
        `${this.baseUrl}/api/search?q=${encodeURIComponent(query)}&limit=${limit}`,
        `${this.baseUrl}/search?q=${encodeURIComponent(query)}&limit=${limit}`,
        `${this.baseUrl}/docs/search?q=${encodeURIComponent(query)}&limit=${limit}`
      ];

      let attemptIndex = 0;
      
      const tryNextUrl = () => {
        if (attemptIndex >= searchUrls.length) {
          resolve(null);
          return;
        }

        const url = searchUrls[attemptIndex];
        attemptIndex++;

        const req = https.get(url, (res) => {
          let data = '';
          
          res.on('data', (chunk) => {
            data += chunk;
          });
          
          res.on('end', () => {
            if (res.statusCode === 200) {
              try {
                const results = JSON.parse(data);
                resolve(results);
              } catch (e) {
                tryNextUrl();
              }
            } else {
              tryNextUrl();
            }
          });
        });

        req.on('error', () => {
          tryNextUrl();
        });

        req.setTimeout(5000, () => {
          req.destroy();
          tryNextUrl();
        });
      };

      tryNextUrl();
    });
  }

  /**
   * Search local documentation files
   * @param {string} query - The search query
   * @param {number} limit - Number of results to return
   * @returns {Promise<Array>} Search results
   */
  async searchLocal(query, limit) {
    const results = [];
    const queryLower = query.toLowerCase();
    
    try {
      // Read docs.json to get all pages
      const docsJson = JSON.parse(fs.readFileSync(path.join(this.localDocsPath, 'docs.json'), 'utf8'));
      const pages = this.extractPagesFromNavigation(docsJson.navigation);
      
      // Search through each page
      for (const page of pages) {
        const filePath = path.join(this.localDocsPath, `${page}.mdx`);
        
        // Try both .mdx and .md extensions
        let content = '';
        if (fs.existsSync(filePath)) {
          content = fs.readFileSync(filePath, 'utf8');
        } else {
          const mdPath = path.join(this.localDocsPath, `${page}.md`);
          if (fs.existsSync(mdPath)) {
            content = fs.readFileSync(mdPath, 'utf8');
          }
        }
        
        if (content) {
          const contentLower = content.toLowerCase();
          if (contentLower.includes(queryLower)) {
            // Extract title from frontmatter or first heading
            const title = this.extractTitle(content, page);
            
            // Find relevant snippets
            const snippets = this.extractSnippets(content, queryLower, 3);
            
            results.push({
              title,
              url: `/docs/${page}`,
              path: page,
              snippets,
              relevance: this.calculateRelevance(content, queryLower)
            });
          }
        }
      }
      
      // Sort by relevance and limit results
      results.sort((a, b) => b.relevance - a.relevance);
      return results.slice(0, limit);
      
    } catch (error) {
      console.error('Local search error:', error.message);
      return [];
    }
  }

  /**
   * Extract all pages from navigation structure
   * @param {Object} navigation - Navigation object from docs.json
   * @returns {Array} Array of page paths
   */
  extractPagesFromNavigation(navigation) {
    const pages = [];
    
    const extractPages = (items) => {
      if (Array.isArray(items)) {
        items.forEach(item => {
          if (typeof item === 'string') {
            pages.push(item);
          } else if (item.pages) {
            extractPages(item.pages);
          }
        });
      }
    };
    
    navigation.groups.forEach(group => {
      extractPages(group.pages);
    });
    
    return pages;
  }

  /**
   * Extract title from content
   * @param {string} content - File content
   * @param {string} fallback - Fallback title
   * @returns {string} Extracted title
   */
  extractTitle(content, fallback) {
    // Try to extract from frontmatter
    const frontmatterMatch = content.match(/^---\s*\n(.*?)\n---/s);
    if (frontmatterMatch) {
      const titleMatch = frontmatterMatch[1].match(/title:\s*["'](.+?)["']/);
      if (titleMatch) {
        return titleMatch[1];
      }
    }
    
    // Try to extract from first heading
    const headingMatch = content.match(/^#\s+(.+)/m);
    if (headingMatch) {
      return headingMatch[1];
    }
    
    // Use fallback
    return fallback.split('/').pop().replace(/[-_]/g, ' ');
  }

  /**
   * Extract relevant snippets containing the search query
   * @param {string} content - File content
   * @param {string} query - Search query (lowercase)
   * @param {number} maxSnippets - Maximum number of snippets
   * @returns {Array} Array of relevant snippets
   */
  extractSnippets(content, query, maxSnippets) {
    const snippets = [];
    const lines = content.split('\n');
    
    for (let i = 0; i < lines.length && snippets.length < maxSnippets; i++) {
      const line = lines[i];
      if (line.toLowerCase().includes(query)) {
        // Get context around the match
        const start = Math.max(0, i - 1);
        const end = Math.min(lines.length, i + 2);
        const snippet = lines.slice(start, end).join('\n').trim();
        
        if (snippet.length > 0) {
          snippets.push(snippet);
        }
      }
    }
    
    return snippets;
  }

  /**
   * Calculate relevance score
   * @param {string} content - File content
   * @param {string} query - Search query (lowercase)
   * @returns {number} Relevance score
   */
  calculateRelevance(content, query) {
    const contentLower = content.toLowerCase();
    const queryWords = query.split(/\s+/);
    
    let score = 0;
    queryWords.forEach(word => {
      const matches = (contentLower.match(new RegExp(word, 'g')) || []).length;
      score += matches;
    });
    
    return score;
  }

  /**
   * Format search results for display
   * @param {Array} results - Search results
   * @returns {string} Formatted results
   */
  formatResults(results) {
    if (!results || results.length === 0) {
      return 'No results found.';
    }

    let output = `Found ${results.length} result(s):\n\n`;
    
    results.forEach((result, index) => {
      output += `${index + 1}. **${result.title}**\n`;
      output += `   URL: https://flatfile.com${result.url}\n`;
      if (result.snippets && result.snippets.length > 0) {
        output += `   Preview: ${result.snippets[0].substring(0, 150)}...\n`;
      }
      output += '\n';
    });
    
    return output;
  }
}

// CLI interface
async function main() {
  const args = process.argv.slice(2);
  
  if (args.length === 0) {
    console.log('Usage: node flatfile-docs-search.js <search-query> [limit]');
    console.log('Example: node flatfile-docs-search.js "authentication" 5');
    process.exit(1);
  }
  
  const query = args[0];
  const limit = parseInt(args[1]) || 10;
  
  const search = new FlatfileDocsSearch();
  
  console.log(`Searching for: "${query}"`);
  console.log('---');
  
  const results = await search.search(query, limit);
  console.log(search.formatResults(results));
}

// Export for use as module
module.exports = FlatfileDocsSearch;

// Run as CLI if called directly
if (require.main === module) {
  main().catch(console.error);
}


#!/usr/bin/env node

/**
 * Populate Search Results Script
 * 
 * This script reads the flatfile-terms.csv file and populates the search results
 * by actually searching the Flatfile docs website for each term.
 */

const fs = require('fs');
const https = require('https');
const path = require('path');

class FlatfileSearchPopulator {
  constructor() {
    this.baseUrl = 'https://flatfile.com';
    this.csvPath = path.join(__dirname, '..', 'data', 'flatfile-terms.csv');
    this.outputPath = path.join(__dirname, '..', 'data', 'flatfile-terms-populated.csv');
    this.delay = 1000; // 1 second delay between requests to be respectful
  }

  /**
   * Read CSV file and parse it
   * @returns {Array} Array of term objects
   */
  readCSV() {
    try {
      const content = fs.readFileSync(this.csvPath, 'utf8');
      const lines = content.trim().split('\n');
      const headers = lines[0].split(',');
      
      const terms = [];
      for (let i = 1; i < lines.length; i++) {
        const values = this.parseCSVLine(lines[i]);
        const term = {};
        headers.forEach((header, index) => {
          term[header] = values[index] || '';
        });
        terms.push(term);
      }
      
      return terms;
    } catch (error) {
      console.error('Error reading CSV:', error.message);
      return [];
    }
  }

  /**
   * Parse a CSV line handling quoted values
   * @param {string} line - CSV line
   * @returns {Array} Parsed values
   */
  parseCSVLine(line) {
    const values = [];
    let current = '';
    let inQuotes = false;
    
    for (let i = 0; i < line.length; i++) {
      const char = line[i];
      
      if (char === '"' && (i === 0 || line[i-1] === ',')) {
        inQuotes = true;
      } else if (char === '"' && inQuotes && (i === line.length - 1 || line[i+1] === ',')) {
        inQuotes = false;
      } else if (char === ',' && !inQuotes) {
        values.push(current.trim());
        current = '';
      } else {
        current += char;
      }
    }
    
    values.push(current.trim());
    return values;
  }

  /**
   * Search for a term on the Flatfile docs website
   * @param {string} term - The search term
   * @returns {Promise<Object>} Search result object
   */
  async searchFlatfileDocs(term) {
    return new Promise((resolve) => {
      // Try to search by accessing the docs site and looking for the term
      const searchUrl = `${this.baseUrl}/docs`;
      
      const req = https.get(searchUrl, (res) => {
        let data = '';
        
        res.on('data', (chunk) => {
          data += chunk;
        });
        
        res.on('end', () => {
          try {
            // Parse the HTML to find navigation or search results
            const result = this.parseDocsPage(data, term);
            resolve(result);
          } catch (error) {
            console.error(`Error parsing docs page for term "${term}":`, error.message);
            resolve({
              title: 'Error',
              url: '',
              snippet: 'Failed to search'
            });
          }
        });
      });

      req.on('error', (error) => {
        console.error(`Error searching for term "${term}":`, error.message);
        resolve({
          title: 'Error',
          url: '',
          snippet: 'Network error'
        });
      });

      req.setTimeout(10000, () => {
        req.destroy();
        resolve({
          title: 'Timeout',
          url: '',
          snippet: 'Request timeout'
        });
      });
    });
  }

  /**
   * Parse the docs page HTML to find relevant content
   * @param {string} html - HTML content
   * @param {string} term - Search term
   * @returns {Object} Search result
   */
  parseDocsPage(html, term) {
    const termLower = term.toLowerCase();
    
    // Look for navigation items that match the term
    const navMatches = this.findInNavigation(html, termLower);
    if (navMatches.length > 0) {
      return navMatches[0];
    }
    
    // Look for content that mentions the term
    const contentMatches = this.findInContent(html, termLower);
    if (contentMatches.length > 0) {
      return contentMatches[0];
    }
    
    // Return a default result if nothing found
    return {
      title: 'Not found',
      url: '',
      snippet: `No direct match found for "${term}"`
    };
  }

  /**
   * Find term in navigation elements
   * @param {string} html - HTML content
   * @param {string} term - Search term (lowercase)
   * @returns {Array} Array of matches
   */
  findInNavigation(html, term) {
    const matches = [];
    
    // Look for navigation links that contain the term
    const linkRegex = /<a[^>]*href=["']([^"']*docs[^"']*)["'][^>]*>([^<]*)<\/a>/gi;
    let match;
    
    while ((match = linkRegex.exec(html)) !== null) {
      const url = match[1];
      const linkText = match[2].trim();
      
      if (linkText.toLowerCase().includes(term)) {
        matches.push({
          title: linkText,
          url: url.startsWith('http') ? url : `https://flatfile.com${url}`,
          snippet: `Navigation link: ${linkText}`
        });
      }
    }
    
    return matches;
  }

  /**
   * Find term in page content
   * @param {string} html - HTML content
   * @param {string} term - Search term (lowercase)
   * @returns {Array} Array of matches
   */
  findInContent(html, term) {
    const matches = [];
    
    // Look for headings that contain the term
    const headingRegex = /<h[1-6][^>]*>([^<]*)<\/h[1-6]>/gi;
    let match;
    
    while ((match = headingRegex.exec(html)) !== null) {
      const heading = match[1].trim();
      
      if (heading.toLowerCase().includes(term)) {
        matches.push({
          title: heading,
          url: 'https://flatfile.com/docs',
          snippet: `Heading: ${heading}`
        });
      }
    }
    
    return matches;
  }

  /**
   * Use local search as fallback
   * @param {string} term - Search term
   * @returns {Promise<Object>} Search result
   */
  async searchLocalFallback(term) {
    try {
      const AdvancedFlatfileDocsSearch = require('./advanced-search.js');
      const search = new AdvancedFlatfileDocsSearch();
      
      const results = await search.search(term, { limit: 1 });
      
      if (results && results.length > 0) {
        const result = results[0];
        return {
          title: result.title,
          url: `https://flatfile.com${result.url}`,
          snippet: result.description || (result.snippets && result.snippets[0]) || 'Local search result'
        };
      }
    } catch (error) {
      console.error(`Local search failed for "${term}":`, error.message);
    }
    
    return {
      title: 'Not found',
      url: '',
      snippet: 'No results found'
    };
  }

  /**
   * Write results to CSV
   * @param {Array} terms - Array of term objects with populated results
   */
  writeCSV(terms) {
    try {
      const headers = ['term', 'search_result_title', 'search_result_url', 'search_result_snippet'];
      const csvContent = [headers.join(',')];
      
      terms.forEach(term => {
        const row = [
          this.escapeCSVField(term.term),
          this.escapeCSVField(term.search_result_title),
          this.escapeCSVField(term.search_result_url),
          this.escapeCSVField(term.search_result_snippet)
        ];
        csvContent.push(row.join(','));
      });
      
      fs.writeFileSync(this.outputPath, csvContent.join('\n'));
      console.log(`\n✅ Results written to: ${this.outputPath}`);
    } catch (error) {
      console.error('Error writing CSV:', error.message);
    }
  }

  /**
   * Escape CSV field (add quotes if needed)
   * @param {string} field - Field value
   * @returns {string} Escaped field
   */
  escapeCSVField(field) {
    if (!field) return '';
    
    // Add quotes if field contains comma, quote, or newline
    if (field.includes(',') || field.includes('"') || field.includes('\n')) {
      return `"${field.replace(/"/g, '""')}"`;
    }
    
    return field;
  }

  /**
   * Add delay between requests
   * @param {number} ms - Milliseconds to wait
   */
  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  /**
   * Main function to populate search results
   */
  async run() {
    console.log('🔍 Populating Flatfile docs search results...\n');
    
    const terms = this.readCSV();
    console.log(`📋 Found ${terms.length} terms to search`);
    
    let processed = 0;
    
    for (const term of terms) {
      console.log(`\n[${processed + 1}/${terms.length}] Searching for: "${term.term}"`);
      
      try {
        // Try web search first
        let result = await this.searchFlatfileDocs(term.term);
        
        // If web search didn't find anything useful, try local search
        if (result.title === 'Not found' || result.title === 'Error') {
          console.log('  → Web search failed, trying local search...');
          result = await this.searchLocalFallback(term.term);
        }
        
        // Populate the term object with search results
        term.search_result_title = result.title;
        term.search_result_url = result.url;
        term.search_result_snippet = result.snippet;
        
        console.log(`  ✅ Found: ${result.title}`);
        
        // Add delay to be respectful to the server
        if (processed < terms.length - 1) {
          await this.sleep(this.delay);
        }
        
      } catch (error) {
        console.error(`  ❌ Error searching for "${term.term}":`, error.message);
        term.search_result_title = 'Error';
        term.search_result_url = '';
        term.search_result_snippet = error.message;
      }
      
      processed++;
    }
    
    // Write results to new CSV file
    this.writeCSV(terms);
    
    console.log('\n🎉 Search population completed!');
    console.log(`📊 Processed ${processed} terms`);
    console.log(`📁 Results saved to: ${this.outputPath}`);
  }
}

// CLI interface
async function main() {
  const args = process.argv.slice(2);
  
  if (args.includes('--help') || args.includes('-h')) {
    console.log('Usage: node populate-search-results.js [options]');
    console.log('');
    console.log('Options:');
    console.log('  --help, -h     Show this help message');
    console.log('  --delay <ms>   Delay between requests in milliseconds (default: 1000)');
    console.log('');
    console.log('This script reads flatfile-terms.csv and populates search results');
    console.log('by searching the Flatfile docs website for each term.');
    console.log('');
    console.log('Output: flatfile-terms-populated.csv');
    return;
  }
  
  const populator = new FlatfileSearchPopulator();
  
  // Parse delay option
  const delayIndex = args.indexOf('--delay');
  if (delayIndex !== -1 && args[delayIndex + 1]) {
    const delay = parseInt(args[delayIndex + 1]);
    if (!isNaN(delay) && delay > 0) {
      populator.delay = delay;
      console.log(`⏱️  Using custom delay: ${delay}ms`);
    }
  }
  
  await populator.run();
}

// Export for use as module
module.exports = FlatfileSearchPopulator;

// Run as CLI if called directly
if (require.main === module) {
  main().catch(console.error);
}


#!/usr/bin/env node

/**
 * Test script for the populate search results functionality
 * Tests with just a few terms to verify everything works
 */

const fs = require('fs');
const path = require('path');

// Create a small test CSV
const testCSV = `term,search_result_title,search_result_url,search_result_snippet
Authentication,,,
Blueprint,,,
Webhook,,,
React,,,
Plugin,,,`;

const testPath = path.join(__dirname, '..', 'data', 'test-terms.csv');
fs.writeFileSync(testPath, testCSV);

console.log('📝 Created test CSV with 5 terms');

// Modify the populator to use test file
const FlatfileSearchPopulator = require('../src/populate-search-results.js');

class TestPopulator extends FlatfileSearchPopulator {
  constructor() {
    super();
    this.csvPath = testPath;
    this.outputPath = path.join(__dirname, '..', 'data', 'test-terms-populated.csv');
    this.delay = 500; // Shorter delay for testing
  }
}

async function runTest() {
  console.log('🧪 Running test population...\n');
  
  const populator = new TestPopulator();
  await populator.run();
  
  // Clean up test file
  fs.unlinkSync(testPath);
  
  console.log('\n✅ Test completed! Check test-terms-populated.csv for results.');
}

runTest().catch(console.error);


#!/usr/bin/env node

/**
 * Test script for Flatfile Docs Search Tools
 * 
 * This script demonstrates the functionality of the search tools
 * and can be used to verify they're working correctly.
 */

const AdvancedFlatfileDocsSearch = require('../src/advanced-search.js');

async function runTests() {
  console.log('🔍 Testing Flatfile Docs Search Tools\n');
  
  const search = new AdvancedFlatfileDocsSearch();
  
  // Test 1: Basic search
  console.log('Test 1: Basic search for "authentication"');
  console.log('=' .repeat(50));
  
  try {
    const results1 = await search.search('authentication', { limit: 3 });
    console.log(`✅ Found ${results1.length} results`);
    results1.forEach((result, i) => {
      console.log(`${i + 1}. ${result.title} (relevance: ${result.relevance.toFixed(2)})`);
    });
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n');
  
  // Test 2: Category filtering
  console.log('Test 2: Search with category filter');
  console.log('=' .repeat(50));
  
  try {
    const results2 = await search.search('webhook', { 
      limit: 3, 
      category: 'guides',
      sortBy: 'relevance' 
    });
    console.log(`✅ Found ${results2.length} results in 'guides' category`);
    results2.forEach((result, i) => {
      console.log(`${i + 1}. ${result.title} (${result.category})`);
    });
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n');
  
  // Test 3: Show categories
  console.log('Test 3: Available categories');
  console.log('=' .repeat(50));
  
  try {
    const categories = search.getAvailableCategories();
    console.log(`✅ Found ${categories.length} categories:`);
    categories.forEach(cat => {
      console.log(`  - ${cat.name} (${cat.key})`);
    });
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n');
  
  // Test 4: Search with content inclusion
  console.log('Test 4: Search with content inclusion');
  console.log('=' .repeat(50));
  
  try {
    const results4 = await search.search('actions', { 
      limit: 1, 
      includeContent: true 
    });
    console.log(`✅ Found ${results4.length} results with content`);
    if (results4.length > 0) {
      console.log(`Content length: ${results4[0].content ? results4[0].content.length : 0} characters`);
      console.log(`Headings: ${results4[0].headings ? results4[0].headings.length : 0}`);
    }
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n');
  
  // Test 5: Different sorting methods
  console.log('Test 5: Different sorting methods');
  console.log('=' .repeat(50));
  
  try {
    const results5 = await search.search('plugin', { 
      limit: 5, 
      sortBy: 'title' 
    });
    console.log(`✅ Found ${results5.length} results sorted by title`);
    results5.forEach((result, i) => {
      console.log(`${i + 1}. ${result.title}`);
    });
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n');
  
  // Test 6: Performance test
  console.log('Test 6: Performance test (cached vs uncached)');
  console.log('=' .repeat(50));
  
  try {
    const startTime1 = Date.now();
    await search.search('authentication', { limit: 5 });
    const time1 = Date.now() - startTime1;
    
    const startTime2 = Date.now();
    await search.search('authentication', { limit: 5 });
    const time2 = Date.now() - startTime2;
    
    console.log(`✅ First search: ${time1}ms`);
    console.log(`✅ Cached search: ${time2}ms`);
    console.log(`✅ Performance improvement: ${((time1 - time2) / time1 * 100).toFixed(1)}%`);
  } catch (error) {
    console.log(`❌ Error: ${error.message}`);
  }
  
  console.log('\n🎉 All tests completed!');
}

// Run tests if called directly
if (require.main === module) {
  runTests().catch(console.error);
}

module.exports = runTests;


An SDK wrapper that enables you to seamlessly integrate a secure and user-friendly data import experience into your client-side application

@flatfile/javascript

**Welcome to the new Flatfile Developer Docs!** &nbsp; Learn more about the new docs [here](https://flatfile.com/docs/welcome-to-new-docs).

Flatfile Developer Docs

Welcome to Flatfile

Get up and running with Flatfile in minutes using AutoBuild to create a complete data import solution

Getting Started with AutoBuild

Running through the import flow with your Flatfile App

Creating Your First Project

Understanding Flatfile's building blocks and architecture

Overview

Use Environments for testing and authentication

Environments

Apps

Micro-applications for content and data storage

Spaces

Define your data schema to structure exactly how data should look, behave, and connect

Blueprints

Standalone webpages within Flatfile Spaces for guidance and dynamic content

Documents

Database-like containers with type-strict Blueprints for data import

Workbooks

Individual data tables within Workbooks that organize and structure imported data

Sheets

Blueprint definitions that define the structure and validation rules for your data in Flatfile

Fields

Individual data rows that represent your imported data in Flatfile

Records

The anatomy of Events and Listeners - core components of the Flatfile Platform

Events and Listeners

Discrete tasks executed asynchronously in response to events

Jobs

Actions

Reusable functionality modules that extend Flatfile's capabilities

Plugins

Extend your Flatfile data engine with powerful plugins for extraction, transformation, validation, and integration

Automatically convert data in a Flatfile Sheet to match the data types defined in the corresponding Blueprint

Autocast Plugin

Automatically map columns for headless data import workflows in Flatfile with configurable confidence levels

Automap Plugin

Comprehensive boolean validation plugin for Flatfile that handles various representations of boolean values with multi-language support and flexible configuration options.

Boolean Validator

Extend Flatfile validation capabilities with custom validation logic for complex field and sheet-level constraints

Constraints Plugin

Automatically converts currency values from a source currency to a target currency using Open Exchange Rates API with support for historical exchange rates.

Currency Conversion Plugin

Automatically parse and standardize date values during data import, converting various date formats into a consistent output format.

Date Format Normalizer

A Flatfile plugin that provides functionality to find and remove duplicate records from a sheet based on specified field values or custom logic.

Deduplicate Sheet Records

Export data from all sheets within a Flatfile Workbook into delimited text files and compress them into a single ZIP archive for download.

Delimited File Zip Exporter

Parse text files with custom delimiters and automatically extract structured data for import into Flatfile

Delimiter Extractor Plugin

Validate email addresses in Flatfile data imports with format checking, required field validation, and disposable domain blocking

Email Validation Plugin

A server-side utility for Flatfile that allows users to export data from an entire Flatfile Workbook into a single, downloadable Microsoft Excel (.xlsx) file.

Export Workbook Plugin

Automatically extract data from Microsoft SQL Server backup files (.bak) into Flatfile Workbooks for easy data review and processing.

Foreign Database Extractor for Microsoft SQL Server

Automatically enrich location data using the Google Maps Geocoding API to convert addresses into geographic coordinates and vice-versa.

Geocode Address Data

Automatically processes GPX (GPS Exchange Format) files to extract waypoints, tracks, routes, and calculate geographic statistics like distance and elevation gain.

GPX File Parser and Analyzer

Automatically generate Flatfile Space configurations by converting GraphQL schemas into Workbooks and Sheets, streamlining data import setup for GraphQL APIs.

GraphQL Schema to Flatfile Blueprint Converter

Parse HTML files and extract data from tables within them, converting structured data into Flatfile-compatible format

HTML Table Extractor

A Flatfile plugin that validates and formats International Standard Book Number (ISBN) data during the import process, supporting both ISBN-10 and ISBN-13 standards with automatic formatting and conversion capabilities.

ISBN Validator Plugin

A Flatfile plugin that streamlines the handling of asynchronous Jobs by managing their lifecycle, progress reporting, and completion status.

Job Handler Plugin

Parse JSON and JSON Lines files uploaded to Flatfile and extract data into Sheets within a Workbook

JSON Extractor

Automatically transform JSON Schema into Flatfile Blueprint for streamlined data model setup

JSON Schema Converter

Automatically parse Markdown files and extract tables into Flatfile Sheets

Markdown File Extractor

Connect Flatfile to the Merge.dev unified API platform to sync data from hundreds of third-party integrations including HRIS, ATS, CRM, and Accounting systems.

Merge.dev Connection Plugin

Comprehensive validation for numeric data during the Flatfile import process with support for ranges, formats, and special number types.

Number Validation Plugin

Automatically converts OpenAPI v3.0.3 schemas into Flatfile Blueprints to streamline Space setup using existing API data structures

OpenAPI Schema to Flatfile Blueprint Converter

Extract tabular data from PDF files and convert them to CSV format using the pdftables.com service

PDF Extractor Plugin

Validates and formats phone numbers in Flatfile using country-specific validation with libphonenumber-js library

Phone Number Validation and Formatting Plugin

A Flatfile plugin that performs data analysis and summarization by generating pivot tables from workbook data and uploading them as Markdown documents.

Pivot Table Exporter

Execute custom logic on individual data records within a Flatfile Sheet with support for validation, transformation, enrichment, and cleaning.

Record Hook Plugin

Automatically apply schema changes to existing, live workbooks when a new version of a Flatfile Agent is deployed

Rollout: Automatic Workbook Updates

Automatically analyze sentiment in text fields and add sentiment scores and categories to your Flatfile records

Sentiment Analysis Plugin

Automatically configure new Flatfile Spaces by cloning an existing Space Template, including workbooks, documents, and settings.

Space Configure from Template

Programmatically set up and configure a new Flatfile Space with workbooks, sheets, documents, and metadata from a single configuration object.

Flatfile Space Configuration Plugin

Automatically create Flatfile Blueprints from SQL Data Definition Language (DDL) files to streamline database schema imports.

SQL DDL to Flatfile Blueprint Converter

Automatically applies server-side validation rules to records during the data import process using reusable JavaScript functions defined at the App level.

Stored Constraints Plugin

A comprehensive plugin for validating and transforming string data during the Flatfile import process with configurable validation rules, pattern matching, and automatic data corrections.

String Validator Plugin

Automatically enriches Flatfile data by performing text summarization and key phrase extraction on specified text fields using natural language processing.

Text Summarization and Key Phrase Extraction Plugin

Automatically translate text content in specified fields using Google Translate API and create new fields with translated content

Field Translation using Google Translate API

Automatically hides unmapped columns after the mapping stage to provide a cleaner data review experience

View Mapped Data Plugin

Send workbook data from Flatfile to external webhook endpoints for seamless integration with your systems

Webhook Egress

Forward all Flatfile events to a webhook URL for language-agnostic integration with external systems

Webhook Event Forwarder

Automatically convert What3Words addresses into standard geographic data including country codes, nearest places, and latitude/longitude coordinates using the official What3Words API.

What3Words Converter Plugin

Parse various Excel file formats (.xls, .xlsx, .xlsm, .xlsb, .xltx, .xltm) and extract structured data with support for header detection, merged cells, and hierarchical spreadsheets.

Excel Extractor

A Flatfile plugin for parsing XML files, flattening nested structures and attributes, and converting them into tabular format for easy import into Flatfile Sheets.

XML Extractor

Automates the creation of a Flatfile Blueprint (Workbook and Sheets) from one or more YAML schema definitions, streamlining the setup of a Flatfile Space by converting existing data models into a Flatfile-compatible format.

YAML Schema to Flatfile Blueprint Converter

Automatically decompress .zip files uploaded to a Flatfile Space and extract their contents as individual files

Zip Extractor

Comprehensive guides to help you master Flatfile's features and capabilities

Guides Overview

Listeners are the core of Flatfile's event-driven architecture. They respond to events throughout the data import lifecycle, from space configuration to data validation and job processing. This guide will help you understand how to create and organize listeners effectively.

Setting Up Your First Listener

Complete guide to authenticating with Flatfile using API keys, Personal Access Tokens, and managing roles and permissions

Authentication and Authorization

Trigger custom operations based on user input

Using Record Hooks

Accepting Additional Fields

Learn how to use Flatfile's Advanced Filters to efficiently filter and search through your data

Advanced Filters

Build custom file processing plugins to handle unique data formats and transform files into structured data

Custom Extractors

Learn how to customize the look and feel of Flatfile to match your brand

Theming

Translate and customize your Space with text overrides

Internationalization

Get data out of Flatfile using various transmission methods and file formats

Data Egress

Store descriptive information or data that provides additional context

Metadata

Multi-Part Jobs

Narrow down the scope of Spaces, Workbooks, and Sheets

Namespaces

Secrets

Embed Flatfile data import directly into your application

JavaScript Embedding

React Embedding

Angular Embedding

Vue Embedding

Authentication and credentials for embedded Flatfile

Authentication

Complete reference for all Flatfile events, their payloads, and when they are triggered

Event Reference

Icon Reference

Command line interface for developing, deploying, and managing Flatfile Agents

CLI Reference

Learn how to filter data in Sheets with FFQL.

Flatfile Query Language

AI-optimized documentation formats and tools for better LLM integration

For LLMs

Using the V2 Records API with the Flatfile TypeScript API package

V2 Records API

Explore our guides and examples to implement Flatfile.

A selection of example projects to get started with.

Example Projects

Notable additions and updates to the Flatfile platform

Platform

Notable additions and updates to Flatfile core libraries

SDKs

Notable additions and updates to Flatfile plugins

The Flatfile CLI is a command-line tool that simplifies the integration process with Flatfile by providing developers with commands to manage and configure their integration from their local environment.

flatfile

Listener combines event handling capabilities with Flatfile-specific functionality, such as working with Flatfile records and sessions. Simply put, it receives events and it responds to events.

@flatfile/listener

@flatfile/react

@flatfile/angular

@flatfile/vue

A plugin for parsing an XLSX file and extracting it into Flatfile.

@flatfile/plugin-xlsx-extractor

A plugin for parsing a JSON file and extracting it into Flatfile.

@flatfile/plugin-json-extractor

A plugin for parsing an XML file and extracting it into Flatfile.

@flatfile/plugin-xml-extractor

A plugin for decompressing a zip file and uploading the decompressed files into Flatfile.

Getting Started

Core Concepts

Plugins

Advanced Guides

Embedding Flatfile

Reference

@flatfile/javascript

Subscribe to updates via RSS