[server] Add retry while putting metadata obj in s3

server/pkg/controller/filedata/s3.go

@@ -15,6 +15,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"io"
 	"os"
+	"strings"
 	stime "time"
 )
 
@@ -86,10 +87,18 @@ func (c *Controller) uploadObject(obj fileData.S3FileMetadata, objectKey string,
 		Key:    &objectKey,
 		Body:   bytes.NewReader(embeddingObj),
 	}
-	result, err := uploader.Upload(&up)
+	var err error
+	var result *s3manager.UploadOutput
+	for retries := 0; retries < 3; retries++ {
+		result, err = uploader.Upload(&up)
+		if err == nil || !strings.Contains(err.Error(), "connection reset by peer") {
+			break
+		}
+		stime.Sleep(50 * stime.Millisecond)
+	}
 	if err != nil {
 		log.Error(err)
-		return -1, stacktrace.Propagate(err, "")
+		return -1, stacktrace.Propagate(err, "metadata upload failed")
 	}
 	log.Infof("Uploaded to bucket %s", result.Location)
 	return int64(len(embeddingObj)), nil