perf: ⚡ use points for large lists and refactor methods (#35)

components/components.templ

@@ -64,7 +64,7 @@ func CalculatePages(counts queries.Counts, q queries.Query) PagesString {
 }
 
 
-type TableRows []queries.TableRow
+type TableRows *queries.TableRows
 
 var _aboutPage string;
 
@@ -241,7 +241,7 @@ templ Table(rows TableRows, q queries.Query, counts queries.Counts, pages PagesS
                 </tr>
             </thead>
             <tbody>
-			    for _, row := range rows {
+			    for _, row := range rows.Rows {
                 	<tr class="hover:bg-gray-200 border-0">
                         <td class="text-left">
 							<a class="italic" href={ templ.URL("https://www.gbif.org/species/" + row.TaxonID)} target="_blank">

pkg/gbif/gbif.go

@@ -74,7 +74,7 @@ func UpdateConfig(config Config) {
 }
 
 // FetchLatest fetches the latest observations of a taxon from the GBIF API
-func FetchLatest(taxonID string) []LatestObservation {
+func FetchLatest(taxonID string) *[]LatestObservation {
 	slog.Info("Fetching latest observations from gbif", "taxonID", taxonID)
 	years := getYears(taxonID)
 	if len(years) == 0 {
@@ -84,7 +84,7 @@ func FetchLatest(taxonID string) []LatestObservation {
 	countries := getCountries(taxonID, years)
 
 	baseUrl := endpoint + "?limit=" + fmt.Sprint(limit) + "&" + basisOfRecord + "&" + occurrenceStatus + "&" + "taxonKey=" + taxonID
-	var result []LatestObservation
+	var result = &[]LatestObservation{}
 	for key, year := range countries {
 		i := 0
 		var observations []LatestObservation
@@ -150,7 +150,7 @@ func FetchLatest(taxonID string) []LatestObservation {
 			return observations[b].ObservationDate < observations[a].ObservationDate
 		})
 		if len(observations) > 0 {
-			result = append(result, observations[0])
+			*result = append(*result, observations[0])
 		}
 
 	}
@@ -161,10 +161,10 @@ func FetchLatest(taxonID string) []LatestObservation {
 // SaveObservation saves the latest observation for each taxon
 // It first clears the old observations for each taxon before inserting the new ones
 // to improve performance each insert contains alls new observations for this taxa at once
-func SaveObservation(observation [][]LatestObservation, db *sql.DB) {
-	slog.Info("Updating observations", "taxa", len(observation))
+func SaveObservation(observation *[][]LatestObservation, db *sql.DB) {
+	slog.Info("Updating observations", "taxa", len(*observation))
 	const stmt = "INSERT INTO observations (ObservationID, TaxonID, CountryCode, ObservationDate, ObservationDateOriginal) VALUES"
-	for _, res := range observation {
+	for _, res := range *observation {
 		var insertString []string
 		clearOldObservations(db, res[0].TaxonID)
 		slog.Info("Inserting new for taxaId", "observations", len(res), "taxaId", res[0].TaxonID)

pkg/gbif/gbif_test.go

@@ -33,11 +33,11 @@ func TestFetchLatest(t *testing.T) {
 	if res == nil {
 		t.Errorf("got %v, wanted %v", res, "not nil")
 	}
-	if len(res) == 0 {
-		t.Errorf("got %d, wanted < %d", len(res), 1)
+	if len(*res) == 0 {
+		t.Errorf("got %d, wanted < %d", len(*res), 1)
 	}
-	if res[0].TaxonID != id {
-		t.Errorf("got %s, wanted %s", res[0].TaxonID, id)
+	if (*res)[0].TaxonID != id {
+		t.Errorf("got %s, wanted %s", (*res)[0].TaxonID, id)
 	}
 
 	res = FetchLatest("123456")
@@ -56,8 +56,8 @@ func TestSaveObservations(t *testing.T) {
 		CountryCode:             "AT",
 	}
 
-	var observations [][]LatestObservation
-	observations = append(observations, []LatestObservation{observation})
+	var observations = &[][]LatestObservation{}
+	*observations = append(*observations, []LatestObservation{observation})
 
 	SaveObservation(observations, internal.DB)
 

pkg/queries/queries.go

@@ -50,6 +50,9 @@ type TableRow struct {
 	TaxonFamily  string
 	Taxa         string
 }
+type TableRows struct {
+	Rows []TableRow
+}
 
 var _taxonRankMap = map[string]string{"kingdom": "TaxonKingdom", "phylum": "TaxonPhylum", "class": "TaxonClass", "order": "TaxonOrder", "family": "TaxonFamily"}
 
@@ -112,7 +115,7 @@ func NewQuery(payload any) Query {
 }
 
 // Get the counts of taxa and observations based on the query
-func GetCounts(db *sql.DB, q Query) Counts {
+func (q Query) GetCounts(db *sql.DB) Counts {
 	var err error
 	var taxaCount int
 	var observationCount int
@@ -142,7 +145,7 @@ func GetCounts(db *sql.DB, q Query) Counts {
 }
 
 // Get the table data based on the query
-func GetTableData(db *sql.DB, q Query, increaseLimit ...bool) []TableRow {
+func (q Query) GetTableData(db *sql.DB, increaseLimit ...bool) *TableRows {
 	query := sq.Select(_selectArray...).From("taxa").JoinClause("LEFT OUTER JOIN observations ON observations.TaxonID = taxa.SynonymID").Limit(DefaultPageLimit)
 
 	if increaseLimit != nil && increaseLimit[0] {
@@ -182,7 +185,7 @@ func GetTableData(db *sql.DB, q Query, increaseLimit ...bool) []TableRow {
 
 	rows, err := query.RunWith(db).Query()
 
-	var result []TableRow
+	var result = &TableRows{}
 	if err != nil {
 		slog.Error("Failed to get outdated observations", "error", err)
 		return result
@@ -219,17 +222,17 @@ func GetTableData(db *sql.DB, q Query, increaseLimit ...bool) []TableRow {
 			slog.Error("Failed to get outdated observations", "error", err)
 		}
 
-		result = append(result, row)
+		result.Rows = append(result.Rows, row)
 	}
 
 	return result
 }
 
 // Create a CSV string from the table data, this is used for exporting data
-func CreateCSV(rows []TableRow) string {
+func (rows *TableRows) CreateCSV() string {
 	var csv string
 	csv += strings.Join(_selectArray, ",") + "\n"
-	for _, row := range rows {
+	for _, row := range rows.Rows {
 		scientificName := ""
 		if row.ScientificName.Valid {
 			scientificName = row.ScientificName.String

pkg/queries/queries_test.go

@@ -29,7 +29,7 @@ func TestQuery(t *testing.T) {
 		SHOW_SYNONYMS: false,
 	}
 
-	counts := GetCounts(internal.DB, q)
+	counts := q.GetCounts(internal.DB)
 	if counts.TaxaCount != 1 {
 		t.Errorf("got %d, wanted %d", counts.TaxaCount, 1)
 	}
@@ -38,19 +38,19 @@ func TestQuery(t *testing.T) {
 		t.Errorf("got %d, wanted %d", counts.ObservationCount, 1)
 	}
 
-	table := GetTableData(internal.DB, q)
-	if len(table) != 1 {
-		t.Errorf("got %d, wanted %d", len(table), 1)
+	table := q.GetTableData(internal.DB)
+	if len(table.Rows) != 1 {
+		t.Errorf("got %d, wanted %d", len(table.Rows), 1)
 	}
 
 	q.SHOW_SYNONYMS = true
-	table = GetTableData(internal.DB, q)
-	if len(table) != 2 {
-		t.Errorf("got %d, wanted %d", len(table), 2)
+	table = q.GetTableData(internal.DB)
+	if len(table.Rows) != 2 {
+		t.Errorf("got %d, wanted %d", len(table.Rows), 2)
 	}
 
-	if table[0].TaxonID != DemoTaxa[0] {
-		t.Errorf("got %s, wanted %s", table[0].TaxonID, DemoTaxa[0])
+	if table.Rows[0].TaxonID != DemoTaxa[0] {
+		t.Errorf("got %s, wanted %s", table.Rows[0].TaxonID, DemoTaxa[0])
 	}
 
 }
@@ -83,8 +83,8 @@ func TestConvertCSV(t *testing.T) {
 		SEARCH:        "Urocerus gigas",
 	}
 
-	table := GetTableData(internal.DB, q)
-	csv := CreateCSV(table)
+	table := q.GetTableData(internal.DB)
+	csv := table.CreateCSV()
 	/* Header */
 	if !strings.Contains(csv, "TaxonID,ScientificName") {
 		t.Errorf("got %s, wanted %s", csv, "TaxonID,ScientificName")

scripts/cron/cron.go

@@ -23,17 +23,17 @@ func main() {
 		slog.Info("Fetching observations for outdated taxa", "taxa", ids)
 	}
 
-	var results [][]gbif.LatestObservation
+	var results = &[][]gbif.LatestObservation{}
 	for _, id := range ids {
 		gbif.UpdateLastFetchStatus(internal.DB, id)
 		res := gbif.FetchLatest(id)
 		if res == nil {
 			continue
 		}
-		results = append(results, res)
+		*results = append(*results, *res)
 	}
 
-	if len(results) == 0 {
+	if len(*results) == 0 {
 		slog.Info("No new observations found")
 		return
 	}

server.go

@@ -105,11 +105,11 @@ type Payload struct {
 /* Pages */
 func index(c echo.Context) error {
 	q := buildQuery(c)
-	counts := queries.GetCounts(internal.DB, q)
+	counts := q.GetCounts(internal.DB)
 
 	return render(c,
 		http.StatusAccepted,
-		components.PageTable(queries.GetTableData(internal.DB, q), q, counts, components.CalculatePages(counts, q), cacheBuster))
+		components.PageTable(q.GetTableData(internal.DB), q, counts, components.CalculatePages(counts, q), cacheBuster))
 
 }
 
@@ -127,8 +127,8 @@ func table(c echo.Context) error {
 	q := buildQuery(c)
 	querystring := c.QueryString()
 
-	table := queries.GetTableData(internal.DB, q)
-	counts := queries.GetCounts(internal.DB, q)
+	table := q.GetTableData(internal.DB)
+	counts := q.GetCounts(internal.DB)
 
 	if c.Request().Header.Get("HX-Request") == "" {
 		return c.JSON(http.StatusOK, table)
@@ -169,8 +169,8 @@ func fetch(c echo.Context) error {
 		return c.String(http.StatusNotFound, "No data found")
 	}
 
-	var results [][]gbif.LatestObservation
-	results = append(results, res)
+	var results = &[][]gbif.LatestObservation{}
+	*results = append(*results, *res)
 	gbif.SaveObservation(results, internal.DB)
 
 	c.Response().Header().Set("HX-Trigger", "filterSubmit")
@@ -181,8 +181,8 @@ func fetch(c echo.Context) error {
 func download(c echo.Context) error {
 
 	q := buildQuery(c)
-	table := queries.GetTableData(internal.DB, q, true)
-	csv := queries.CreateCSV(table)
+	table := q.GetTableData(internal.DB, true)
+	csv := table.CreateCSV()
 
 	filename := fmt.Sprintf("extinct-%s-%s.csv", time.Now().Format("2006-01-02"), strings.ReplaceAll(c.QueryString(), "&", "-"))
 
@@ -222,17 +222,17 @@ func cronFetch() {
 	slog.Info("Starting cron")
 
 	ids := gbif.GetOutdatedObservations(internal.DB)
-	var results [][]gbif.LatestObservation
+	var results = &[][]gbif.LatestObservation{}
 	for _, id := range ids {
 		gbif.UpdateLastFetchStatus(internal.DB, id)
 		res := gbif.FetchLatest(id)
 		if res == nil {
 			continue
 		}
-		results = append(results, res)
+		*results = append(*results, *res)
 	}
 
-	if len(results) == 0 {
+	if len(*results) == 0 {
 		slog.Info("No new observations found")
 		return
 	}