From 92380802e9f34bfa6f1647553592edc7e0263f3b Mon Sep 17 00:00:00 2001 From: Nicolai Ort Date: Thu, 1 May 2025 18:11:46 +0200 Subject: [PATCH] perf(cards): Implement generation splitting support for large datasets --- go.mod | 16 +++++-- go.sum | 24 ++++++++++ handlers/card.go | 112 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 132 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 0b4fc0b..3df1eee 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,11 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hhrutter/lzw v1.0.0 // indirect + github.com/hhrutter/pkcs7 v0.2.0 // indirect + github.com/hhrutter/tiff v1.0.2 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/makiuchi-d/gozxing v0.1.1 // indirect @@ -33,7 +36,9 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/oxplot/papersizes v0.0.0-20181201065918-90a3a5ae1915 // indirect + github.com/pdfcpu/pdfcpu v0.10.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/redis/go-redis/v9 v9.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -46,16 +51,19 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/swaggo/files/v2 v2.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.57.0 // indirect + github.com/valyala/fasthttp v1.61.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/image v0.26.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.27.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bb9d46f..69cc789 100644 --- a/go.sum +++ b/go.sum @@ -29,10 +29,18 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0= +github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo= +github.com/hhrutter/pkcs7 v0.2.0 h1:i4HN2XMbGQpZRnKBLsUwO3dSckzgX142TNqY/KfXg+I= +github.com/hhrutter/pkcs7 v0.2.0/go.mod h1:aEzKz0+ZAlz7YaEMY47jDHL14hVWD6iXt0AgqgAvWgE= +github.com/hhrutter/tiff v1.0.2 h1:7H3FQQpKu/i5WaSChoD1nnJbGx4MxU5TlNqqpxw55z8= +github.com/hhrutter/tiff v1.0.2/go.mod h1:pcOeuK5loFUE7Y/WnzGw20YxUdnqjY1P0Jlcieb/cCw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -54,8 +62,12 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/oxplot/papersizes v0.0.0-20181201065918-90a3a5ae1915 h1:4WzMzgExTgBfuUQ/HegMf+jcHtH+c3fl7eySUQUbfzg= github.com/oxplot/papersizes v0.0.0-20181201065918-90a3a5ae1915/go.mod h1:LJRTnhoARxQgMyT7T9L+ZzwR4OrmyHTy5LPxZEzE1CM= +github.com/pdfcpu/pdfcpu v0.10.2 h1:DB2dWuoq0eF0QwHjgyLirYKLTCzFOoZdmmIUSu72aL0= +github.com/pdfcpu/pdfcpu v0.10.2/go.mod h1:Q2Z3sqdRqHTdIq1mPAUl8nfAoim8p3c1ASOaQ10mCpE= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= @@ -99,6 +111,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= +github.com/valyala/fasthttp v1.61.0 h1:VV08V0AfoRaFurP1EWKvQQdPTZHiUzaVoulX1aBDgzU= +github.com/valyala/fasthttp v1.61.0/go.mod h1:wRIV/4cMwUPWnRcDno9hGnYZGh78QzODFfo1LTUhBog= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -111,8 +125,12 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY= +golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= @@ -121,8 +139,12 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= @@ -133,6 +155,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/card.go b/handlers/card.go index 7fc5469..5986f92 100644 --- a/handlers/card.go +++ b/handlers/card.go @@ -1,10 +1,14 @@ package handlers import ( + "fmt" + "os" "slices" "git.odit.services/lfk/document-server/models" "github.com/gofiber/fiber/v2" + "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" ) // GenerateCard godoc @@ -18,7 +22,6 @@ import ( // @Security ApiKeyAuth // @Router /v1/pdfs/cards [post] func (h *DefaultHandler) GenerateCard(c *fiber.Ctx) error { - logger := h.Logger.Named("GenerateCard") cardRequest := new(models.CardRequest) @@ -52,38 +55,115 @@ func (h *DefaultHandler) GenerateCard(c *fiber.Ctx) error { }) } - genConfig := &models.CardTemplateOptions{ - CardSegments: splitCardSegments(cardRequest.Cards), - EventName: h.Config.EventName, - CardSubtitle: h.Config.CardSubtitle, - BarcodeFormat: h.Config.CardBarcodeFormat, - BarcodePrefix: h.Config.CardBarcodePrefix, + segmentLength := calculateOptimalSegmentSize(len(cardRequest.Cards)) + pdfs := []string{} + for i := 0; i < len(cardRequest.Cards); i += segmentLength { + + segment := cardRequest.Cards[i:] + if len(segment) > segmentLength { + segment = cardRequest.Cards[i : i+segmentLength] + } + + genConfig := &models.CardTemplateOptions{ + CardSegments: splitCardSegments(segment), + EventName: h.Config.EventName, + CardSubtitle: h.Config.CardSubtitle, + BarcodeFormat: h.Config.CardBarcodeFormat, + BarcodePrefix: h.Config.CardBarcodePrefix, + } + + logger.Info("Generating card html") + result, err := h.Templater.Execute(template, genConfig) + if err != nil { + logger.Errorw("Error executing template", "error", err) + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": err.Error(), + }) + } + logger.Info("Generated card html") + c.Set(fiber.HeaderContentType, "text/html") + + logger.Info("Converting html to pdf") + pdf, err := h.Converter.ToPdf(result, "a4", false) + if err != nil { + logger.Errorw("Error converting html to pdf", "error", err) + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": err.Error(), + }) + } + + tempFile, err := os.CreateTemp("", fmt.Sprintf("cards-%d-*.pdf", i/segmentLength)) + if err != nil { + logger.Errorw("Error creating temp file", "error", err) + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": err.Error(), + }) + } + defer os.Remove(tempFile.Name()) // Ensure cleanup even on error paths + + if _, err := tempFile.Write(pdf); err != nil { + logger.Errorw("Error writing pdf to temp file", "error", err) + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": err.Error(), + }) + } + tempFile.Close() + pdfs = append(pdfs, tempFile.Name()) } - logger.Info("Generating card html") - result, err := h.Templater.Execute(template, genConfig) + outputFile := "./output.pdf" + conf := model.NewDefaultConfiguration() + err = api.MergeCreateFile(pdfs, outputFile, false, conf) if err != nil { - logger.Errorw("Error executing template", "error", err) + logger.Errorw("Failed to merge PDFs", "error", err) return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ "error": err.Error(), }) } - logger.Info("Generated card html") - c.Set(fiber.HeaderContentType, "text/html") - logger.Info("Converting html to pdf") - pdf, err := h.Converter.ToPdf(result, "a4", false) + // Clean up individual PDF files + for _, file := range pdfs { + if err := os.Remove(file); err != nil { + logger.Warnw("Failed to remove temporary PDF file", "file", file, "error", err) + } + } + + // Set headers and return the merged PDF + c.Set(fiber.HeaderContentType, "application/pdf") + c.Set(fiber.HeaderContentDisposition, "attachment; filename=runner-cards.pdf") + pdfBytes, err := os.ReadFile(outputFile) if err != nil { - logger.Errorw("Error converting html to pdf", "error", err) + logger.Errorw("Failed to read merged PDF", "error", err) return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ "error": err.Error(), }) } + os.Remove(outputFile) // Clean up the merged file + logger.Info("Converted html to pdf") c.Set(fiber.HeaderContentType, "application/pdf") c.Set(fiber.HeaderContentDisposition, "attachment; filename=runner-cards.pdf") - return c.Send(pdf) + return c.Send(pdfBytes) +} + +func calculateOptimalSegmentSize(totalCards int) int { + if totalCards < 30 { + return 25 // Reduces overhead for really small batches + } + + // Base size for small batches + if totalCards < 100 { + return 50 + } + + // For medium batches + if totalCards < 500 { + return 75 + } + + // For large batches, be more conservative + return 100 } func invertCardArrayItemPairs(cards []models.Card) []models.Card {