perf(cards): Implement generation splitting support for large datasets

handlers/card.go

@@ -1,10 +1,14 @@
 package handlers
 
 import (
+	"fmt"
+	"os"
 	"slices"
 
 	"git.odit.services/lfk/document-server/models"
 	"github.com/gofiber/fiber/v2"
+	"github.com/pdfcpu/pdfcpu/pkg/api"
+	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model"
 )
 
 // GenerateCard godoc
@@ -18,7 +22,6 @@ import (
 //	@Security		ApiKeyAuth
 //	@Router			/v1/pdfs/cards [post]
 func (h *DefaultHandler) GenerateCard(c *fiber.Ctx) error {
-
 	logger := h.Logger.Named("GenerateCard")
 
 	cardRequest := new(models.CardRequest)
@@ -52,38 +55,115 @@ func (h *DefaultHandler) GenerateCard(c *fiber.Ctx) error {
 		})
 	}
 
-	genConfig := &models.CardTemplateOptions{
-		CardSegments:  splitCardSegments(cardRequest.Cards),
-		EventName:     h.Config.EventName,
-		CardSubtitle:  h.Config.CardSubtitle,
-		BarcodeFormat: h.Config.CardBarcodeFormat,
-		BarcodePrefix: h.Config.CardBarcodePrefix,
+	segmentLength := calculateOptimalSegmentSize(len(cardRequest.Cards))
+	pdfs := []string{}
+	for i := 0; i < len(cardRequest.Cards); i += segmentLength {
+
+		segment := cardRequest.Cards[i:]
+		if len(segment) > segmentLength {
+			segment = cardRequest.Cards[i : i+segmentLength]
+		}
+
+		genConfig := &models.CardTemplateOptions{
+			CardSegments:  splitCardSegments(segment),
+			EventName:     h.Config.EventName,
+			CardSubtitle:  h.Config.CardSubtitle,
+			BarcodeFormat: h.Config.CardBarcodeFormat,
+			BarcodePrefix: h.Config.CardBarcodePrefix,
+		}
+
+		logger.Info("Generating card html")
+		result, err := h.Templater.Execute(template, genConfig)
+		if err != nil {
+			logger.Errorw("Error executing template", "error", err)
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+				"error": err.Error(),
+			})
+		}
+		logger.Info("Generated card html")
+		c.Set(fiber.HeaderContentType, "text/html")
+
+		logger.Info("Converting html to pdf")
+		pdf, err := h.Converter.ToPdf(result, "a4", false)
+		if err != nil {
+			logger.Errorw("Error converting html to pdf", "error", err)
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+				"error": err.Error(),
+			})
+		}
+
+		tempFile, err := os.CreateTemp("", fmt.Sprintf("cards-%d-*.pdf", i/segmentLength))
+		if err != nil {
+			logger.Errorw("Error creating temp file", "error", err)
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+				"error": err.Error(),
+			})
+		}
+		defer os.Remove(tempFile.Name()) // Ensure cleanup even on error paths
+
+		if _, err := tempFile.Write(pdf); err != nil {
+			logger.Errorw("Error writing pdf to temp file", "error", err)
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+				"error": err.Error(),
+			})
+		}
+		tempFile.Close()
+		pdfs = append(pdfs, tempFile.Name())
 	}
 
-	logger.Info("Generating card html")
-	result, err := h.Templater.Execute(template, genConfig)
+	outputFile := "./output.pdf"
+	conf := model.NewDefaultConfiguration()
+	err = api.MergeCreateFile(pdfs, outputFile, false, conf)
 	if err != nil {
-		logger.Errorw("Error executing template", "error", err)
+		logger.Errorw("Failed to merge PDFs", "error", err)
 		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
 			"error": err.Error(),
 		})
 	}
-	logger.Info("Generated card html")
-	c.Set(fiber.HeaderContentType, "text/html")
 
-	logger.Info("Converting html to pdf")
-	pdf, err := h.Converter.ToPdf(result, "a4", false)
+	// Clean up individual PDF files
+	for _, file := range pdfs {
+		if err := os.Remove(file); err != nil {
+			logger.Warnw("Failed to remove temporary PDF file", "file", file, "error", err)
+		}
+	}
+
+	// Set headers and return the merged PDF
+	c.Set(fiber.HeaderContentType, "application/pdf")
+	c.Set(fiber.HeaderContentDisposition, "attachment; filename=runner-cards.pdf")
+	pdfBytes, err := os.ReadFile(outputFile)
 	if err != nil {
-		logger.Errorw("Error converting html to pdf", "error", err)
+		logger.Errorw("Failed to read merged PDF", "error", err)
 		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
 			"error": err.Error(),
 		})
 	}
+	os.Remove(outputFile) // Clean up the merged file
+
 	logger.Info("Converted html to pdf")
 
 	c.Set(fiber.HeaderContentType, "application/pdf")
 	c.Set(fiber.HeaderContentDisposition, "attachment; filename=runner-cards.pdf")
-	return c.Send(pdf)
+	return c.Send(pdfBytes)
+}
+
+func calculateOptimalSegmentSize(totalCards int) int {
+	if totalCards < 30 {
+		return 25 // Reduces overhead for really small batches
+	}
+
+	// Base size for small batches
+	if totalCards < 100 {
+		return 50
+	}
+
+	// For medium batches
+	if totalCards < 500 {
+		return 75
+	}
+
+	// For large batches, be more conservative
+	return 100
 }
 
 func invertCardArrayItemPairs(cards []models.Card) []models.Card {