Upload filer til en Vektorbutik

/**
 * Upload filer til en Vector Store i batches
 * 
 * @param {string} vectorStoreId - ID for Vector Store
 * @param {FileList|File[]} files - Filer der skal uploades
 * @param {Object} options - Konfigurationsmuligheder
 * @returns {Promise<Array>} - Liste over uploadresultater
 */
async function batchUploadToVectorStore(vectorStoreId, files, options = {}) {
  const {
    batchSize = 5,
    delayBetweenBatches = 2000,
    onProgress = null
  } = options;
  
  const apiKey = 'YOUR_API_KEY';
  const results = [];
  const fileList = Array.from(files);
  const totalBatches = Math.ceil(fileList.length / batchSize);
  
  console.log(`Forbereder at uploade ${fileList.length} filer i ${totalBatches} batches`);
  
  // Behandl filer i batches
  for (let i = 0; i < fileList.length; i += batchSize) {
    const batch = fileList.slice(i, i + batchSize);
    const batchNumber = Math.floor(i / batchSize) + 1;
    
    console.log(`Behandler batch ${batchNumber}/${totalBatches}: ${batch.length} filer`);
    
    if (onProgress) {
      onProgress({
        currentBatch: batchNumber,
        totalBatches: totalBatches,
        filesInBatch: batch.length,
        totalFiles: fileList.length,
        completedFiles: i
      });
    }
    
    // Opret FormData for denne batch
    const formData = new FormData();
    batch.forEach(file => {
      formData.append('files', file);
    });
    
    try {
      const response = await fetch(
        `https://api.rememberizer.ai/api/v1/vector-stores/${vectorStoreId}/documents/upload`,
        {
          method: 'POST',
          headers: {
            'x-api-key': apiKey
          },
          body: formData
        }
      );
      
      if (response.ok) {
        const batchResult = await response.json();
        results.push(batchResult);
        
        console.log(`Batch uploadet med succes - ${batchResult.documents?.length || 0} dokumenter behandlet`);
        
        // Tjek for fejl
        if (batchResult.errors && batchResult.errors.length > 0) {
          console.warn(`Fejl opstået: ${batchResult.errors.length}`);
          batchResult.errors.forEach(error => {
            console.warn(`- ${error.file}: ${error.error}`);
          });
        }
      } else {
        console.error(`Batch upload fejlede med status ${response.status}: ${await response.text()}`);
        results.push({ error: `Batch fejlede med status: ${response.status}` });
      }
    } catch (error) {
      console.error(`Undtagelse under batch upload: ${error.message}`);
      results.push({ error: error.message });
    }
    
    // Tilføj forsinkelse mellem batches for at undgå rate limiting
    if (i + batchSize < fileList.length) {
      console.log(`Pause i ${delayBetweenBatches}ms før næste batch...`);
      await new Promise(resolve => setTimeout(resolve, delayBetweenBatches));
    }
  }
  
  console.log(`Upload fuldført. Behandlet ${fileList.length} filer.`);
  return results;
}

// Eksempel på brug med filinput-element
document.getElementById('upload-button').addEventListener('click', async () => {
  const fileInput = document.getElementById('file-input');
  const vectorStoreId = 'vs_abc123';
  
  const progressBar = document.getElementById('progress-bar');
  
  try {
    const results = await batchUploadToVectorStore(vectorStoreId, fileInput.files, {
      batchSize: 5,
      onProgress: (progress) => {
        // Opdater progress UI
        const percentage = Math.round((progress.completedFiles / progress.totalFiles) * 100);
        progressBar.style.width = `${percentage}%`;
        progressBar.textContent = `${percentage}% (Batch ${progress.currentBatch}/${progress.totalBatches})`;
      }
    });
    
    console.log('Færdige uploadresultater:', results);
  } catch (error) {
    console.error('Upload fejlede:', error);
  }
});

require 'net/http'
require 'uri'
require 'json'
require 'mime/types'

# Upload filer til et Vektorbutik i batcher
#
# @param vector_store_id [String] ID på Vector Store
# @param folder_path [String] Sti til mappen, der indeholder filer, der skal uploades
# @param batch_size [Integer] Antal filer, der skal uploades i hver batch
# @param file_types [Array<String>] Valgfri array af filudvidelser at filtrere efter
# @param delay_between_batches [Float] Sekunder at vente mellem batcher
# @return [Array] Liste over uploadresultater
def batch_upload_to_vector_store(vector_store_id, folder_path, batch_size: 5, file_types: nil, delay_between_batches: 2.0)
  api_key = 'YOUR_API_KEY'
  results = []
  
  # Hent liste over filer i mappen
  files = Dir.entries(folder_path)
    .select { |f| File.file?(File.join(folder_path, f)) }
    .select { |f| file_types.nil? || file_types.include?(File.extname(f).downcase) }
    .map { |f| File.join(folder_path, f) }
  
  puts "Fundet #{files.count} filer til upload"
  total_batches = (files.count.to_f / batch_size).ceil
  
  # Behandl filer i batches
  files.each_slice(batch_size).with_index do |batch, batch_index|
    puts "Behandler batch #{batch_index + 1}/#{total_batches}: #{batch.count} filer"
    
    # Forbered HTTP-anmodningen
    uri = URI("https://api.rememberizer.ai/api/v1/vector-stores/#{vector_store_id}/documents/upload")
    request = Net::HTTP::Post.new(uri)
    request['x-api-key'] = api_key
    
    # Opret en multipart form grænse
    boundary = "RubyBoundary#{rand(1000000)}"
    request['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
    
    # Byg anmodningskroppen
    body = []
    batch.each do |file_path|
      file_name = File.basename(file_path)
      mime_type = MIME::Types.type_for(file_path).first&.content_type || 'application/octet-stream'
      
      begin
        file_content = File.binread(file_path)
        
        body << "--#{boundary}\r\n"
        body << "Content-Disposition: form-data; name=\"files\"; filename=\"#{file_name}\"\r\n"
        body << "Content-Type: #{mime_type}\r\n\r\n"
        body << file_content
        body << "\r\n"
      rescue => e
        puts "Fejl ved læsning af fil #{file_path}: #{e.message}"
      end
    end
    body << "--#{boundary}--\r\n"
    
    request.body = body.join
    
    # Send anmodningen
    begin
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = true
      response = http.request(request)
      
      if response.code.to_i == 200 || response.code.to_i == 201 || response.code.to_i == 207
        batch_result = JSON.parse(response.body)
        results << batch_result
        
        puts "Batch uploadet med succes - #{batch_result['documents']&.count || 0} dokumenter behandlet"
        
        # Tjek for fejl
        if batch_result['errors'] && !batch_result['errors'].empty?
          puts "Fejl opstået: #{batch_result['errors'].count}"
          batch_result['errors'].each do |error|
            puts "- #{error['file']}: #{error['error']}"
          end
        end
      else
        puts "Batch upload mislykkedes med statuskode #{response.code}: #{response.body}"
        results << { "error" => "Batch mislykkedes: #{response.body}" }
      end
    rescue => e
      puts "Undtagelse under batch upload: #{e.message}"
      results << { "error" => e.message }
    end
    
    # Rate limiting - pause mellem batches
    if batch_index < total_batches - 1
      puts "Pauser i #{delay_between_batches} sekunder før næste batch..."
      sleep(delay_between_batches)
    end
  end
  
  puts "Upload fuldført. Behandlet #{files.count} filer."
  results
end

# Eksempel på brug
results = batch_upload_to_vector_store(
  'vs_abc123',
  '/path/to/documents/folder',
  batch_size: 5,
  file_types: ['.pdf', '.docx', '.txt'],
  delay_between_batches: 2.0
)