[server] Expose API to get and put metadata

server/cmd/museum/main.go

@@ -413,6 +413,8 @@ func main() {
 	privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList)
 	privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL)
 	privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl)
+	privateAPI.PUT("/files/data/", fileHandler.PutFileData)
+	privateAPI.POST("files/fetch-data/", fileHandler.GetFilesData)
 
 	privateAPI.POST("/files", fileHandler.CreateOrUpdate)
 	privateAPI.POST("/files/copy", fileHandler.CopyFiles)

server/pkg/api/file.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"github.com/ente-io/museum/pkg/controller/file_copy"
+	"github.com/ente-io/museum/pkg/controller/filedata"
 	"net/http"
 	"os"
 	"strconv"
@@ -24,6 +25,7 @@ import (
 type FileHandler struct {
 	Controller   *controller.FileController
 	FileCopyCtrl *file_copy.FileCopyController
+	FileDataCtrl *filedata.Controller
 }
 
 // DefaultMaxBatchSize is the default maximum API batch size unless specified otherwise

server/pkg/api/file_data.go

@@ -0,0 +1,42 @@
+package api
+
+import (
+	"github.com/ente-io/museum/ente"
+	fileData "github.com/ente-io/museum/ente/filedata"
+	"github.com/ente-io/museum/pkg/utils/handler"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func (f *FileHandler) PutFileData(ctx *gin.Context) {
+	var req fileData.PutFileDataRequest
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error()))
+		return
+	}
+	if err := req.Validate(); err != nil {
+		ctx.JSON(http.StatusBadRequest, err)
+		return
+	}
+	err := f.FileDataCtrl.InsertOrUpdate(ctx, &req)
+	if err != nil {
+		handler.Error(ctx, err)
+
+		return
+	}
+	ctx.JSON(http.StatusOK, gin.H{})
+}
+
+func (f *FileHandler) GetFilesData(ctx *gin.Context) {
+	var req fileData.GetFilesData
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error()))
+		return
+	}
+	resp, err := f.FileDataCtrl.GetFilesData(ctx, req)
+	if err != nil {
+		handler.Error(ctx, err)
+		return
+	}
+	ctx.JSON(http.StatusOK, resp)
+}

server/pkg/api/file_preview.go

@@ -1 +0,0 @@
-package api

server/pkg/controller/filedata/controller.go

@@ -185,7 +185,8 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string)
 			defer func() { <-globalFileFetchSemaphore }() // Release back to global semaphore
 			s3FileMetadata, err := c.fetchS3FileMetadata(context.Background(), row, dc)
 			if err != nil {
-				log.Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err)
+				log.WithField("bucket", row.LatestBucket).
+					Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err)
 				embeddingObjects[i] = bulkS3MetaFetchResult{
 					err:     err,
 					dbEntry: row,
@@ -203,10 +204,10 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string)
 	return embeddingObjects, nil
 }
 
-func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, dc string) (*fileData.S3FileMetadata, error) {
+func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, _ string) (*fileData.S3FileMetadata, error) {
 	opt := _defaultFetchConfig
 	objectKey := row.S3FileMetadataObjectKey()
-	ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", dc)
+	ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", row.LatestBucket)
 	totalAttempts := opt.RetryCount + 1
 	timeout := opt.InitialTimeout
 	for i := 0; i < totalAttempts; i++ {
@@ -222,7 +223,7 @@ func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row,
 			cancel()
 			return nil, stacktrace.Propagate(ctx.Err(), "")
 		default:
-			obj, err := c.downloadObject(fetchCtx, objectKey, dc)
+			obj, err := c.downloadObject(fetchCtx, objectKey, row.LatestBucket)
 			cancel() // Ensure cancel is called to release resources
 			if err == nil {
 				if i > 0 {

server/pkg/utils/s3config/s3config.go

@@ -152,6 +152,10 @@ func (config *S3Config) GetBucket(dc string) *string {
 	return &bucket
 }
 
+func (config *S3Config) IsBucketActive(dc string) bool {
+	return config.buckets[dc] != ""
+}
+
 func (config *S3Config) GetS3Config(dc string) *aws.Config {
 	return config.s3Configs[dc]
 }