feat: Introduce retry mechanism for GET requests

Testing the SLO APIs pagination it was found that it's possible to run
into the same timing issues as Upsert when listing existing configs.
To handle cases where one reply returns a nextPage key/number of configs
that does not match the one of the next reply, GET is now retried.

Author: UnseenWizzard

pkg/rest/config_upload.go

@@ -400,7 +400,7 @@ func getExistingValuesFromEndpoint(client *http.Client, theApi api.Api, urlStrin
 		if isPaginated, nextPage := isPaginatedResponse(objmap); isPaginated {
 			parsedUrl = addNextPageQueryParams(theApi, parsedUrl, nextPage)
 
-			resp, err = get(client, parsedUrl.String(), apiToken)
+			resp, err = getWithRetry(client, parsedUrl.String(), apiToken, 3, 5*time.Second)
 
 			if err != nil {
 				return nil, err
@@ -418,6 +418,33 @@ func getExistingValuesFromEndpoint(client *http.Client, theApi api.Api, urlStrin
 	return existingValues, nil
 }
 
+// getWithRetry works similarly to retry does for PUT and POST
+// this method can be used for API calls we know to have occasional timing issues on GET - e.g. paginated queries that are impacted by replication lag, returning unequal amounts of objects/pages per node
+func getWithRetry(client *http.Client, url string, apiToken string, maxRetries int, timeout time.Duration) (Response, error) {
+	resp, err := get(client, url, apiToken)
+
+	if err == nil && success(resp) {
+		return resp, nil
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		util.Log.Warn("Retrying failed GET request %s after error (HTTP %d): %w", url, resp.StatusCode, err)
+		time.Sleep(timeout)
+		resp, err = get(client, url, apiToken)
+		if err == nil && success(resp) {
+			return resp, err
+		}
+	}
+
+	var retryErr error
+	if err != nil {
+		retryErr = fmt.Errorf("GET request %s failed after %d retries: %w", url, maxRetries, err)
+	} else {
+		retryErr = fmt.Errorf("GET request %s failed after %d retries: (HTTP %d)!\n    Response was: %s", url, maxRetries, resp.StatusCode, resp.Body)
+	}
+	return Response{}, retryErr
+}
+
 func addQueryParamsForNonStandardApis(theApi api.Api, url *url.URL) *url.URL {
 
 	queryParams := url.Query()

pkg/rest/config_upload_test.go

@@ -471,15 +471,51 @@ func Test_GetObjectIdIfAlreadyExists_WorksCorrectlyForAddedQueryParameters(t *te
 				{
 					{"nextPageKey", "page42"},
 				},
+				{
+					{"nextPageKey", "page42"},
+				},
+				{
+					{"nextPageKey", "page42"},
+				},
+				{
+					{"nextPageKey", "page42"},
+				},
 			},
-			expectedApiCalls: 2,
+			expectedApiCalls: 5,
 			serverResponses: []testServerResponse{
 				{200, `{ "nextPageKey": "page42", "values": [ {"id": "1", "name": "name1"} ] }`},
-				{400, `epic fail`},
+				{400, `epic fail`}, // fail paginated request
+				{400, `epic fail`}, // still fail after retry
+				{400, `epic fail`}, // still fail after 2nd retry
+				{400, `epic fail`}, // still fail after 3rd retry
 			},
 			apiKey:      "slo",
 			expectError: true,
 		},
+		{
+			name: "Retries on HTTP error on paginated request and returns eventual success",
+			expectedQueryParamsPerApiCall: [][]testQueryParams{
+				{},
+				{
+					{"nextPageKey", "page42"},
+				},
+				{
+					{"nextPageKey", "page42"},
+				},
+				{
+					{"nextPageKey", "page42"},
+				},
+			},
+			expectedApiCalls: 4,
+			serverResponses: []testServerResponse{
+				{200, `{ "nextPageKey": "page42", "values": [ {"id": "1", "name": "name1"} ] }`},
+				{400, `epic fail`}, // fail paginated request
+				{400, `epic fail`}, // still fail after retry
+				{200, `{ "values": [ {"id": "1", "name": "name1"} ] }`},
+			},
+			apiKey:      "random-api", //not testing a real API, so this won't break if params are ever added to one
+			expectError: false,
+		},
 		{
 			name: "Sends correct param to get all SLOs",
 			expectedQueryParamsPerApiCall: [][]testQueryParams{