diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 7aa2f41c1f..a34276bad4 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -413,12 +413,11 @@ func main() { privateAPI.GET("/files/preview/:fileID", fileHandler.GetThumbnail) privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail) - privateAPI.GET("/files/file-data/playlist/:fileID", fileHandler.GetVideoPlaylist) - privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList) - privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL) - privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl) privateAPI.PUT("/files/data/", fileHandler.PutFileData) - privateAPI.POST("files/fetch-data/", fileHandler.GetFilesData) + privateAPI.POST("files/data/fetch", fileHandler.GetFilesData) + privateAPI.GET("files/data/fetch", fileHandler.GetFileData) + privateAPI.GET("/files/data/preview-upload-url/", fileHandler.GetPreviewUploadURL) + privateAPI.GET("/files/data/preview/", fileHandler.GetPreviewURL) privateAPI.POST("/files", fileHandler.CreateOrUpdate) privateAPI.POST("/files/copy", fileHandler.CopyFiles) diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index f9cfccd28e..9d5f366351 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -5,9 +5,16 @@ import ( "github.com/ente-io/museum/ente" ) +type Entity struct { + FileID int64 `json:"fileID"` + Type ente.ObjectType `json:"type"` + EncryptedData string `json:"encryptedData"` + DecryptionHeader string `json:"decryptionHeader"` +} + // GetFilesData should only be used for getting the preview video playlist and derived metadata. type GetFilesData struct { - FileIDs []int64 `form:"fileIDs" binding:"required"` + FileIDs []int64 `json:"fileIDs" binding:"required"` Type ente.ObjectType `json:"type" binding:"required"` } @@ -24,38 +31,16 @@ func (g *GetFilesData) Validate() error { return nil } -type Entity struct { - FileID int64 `json:"fileID"` - Type ente.ObjectType `json:"type"` - EncryptedData string `json:"encryptedData"` - DecryptionHeader string `json:"decryptionHeader"` +type GetFileData struct { + FileID int64 `form:"fileID" binding:"required"` + Type ente.ObjectType `form:"type" binding:"required"` } -// Row represents the data that is stored in the file_data table. -type Row struct { - FileID int64 - UserID int64 - Type ente.ObjectType - Size int64 - LatestBucket string - ReplicatedBuckets []string - DeleteFromBuckets []string - InflightReplicas []string - PendingSync bool - IsDeleted bool - SyncLockedTill int64 - CreatedAt int64 - UpdatedAt int64 -} - -func (r Row) S3FileMetadataObjectKey() string { - if r.Type == ente.DerivedMeta { - return derivedMetaPath(r.FileID, r.UserID) +func (g *GetFileData) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } - if r.Type == ente.PreviewVideo { - return previewVideoPlaylist(r.FileID, r.UserID) - } - panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) + return nil } type GetFilesDataResponse struct { @@ -73,14 +58,65 @@ type S3FileMetadata struct { Client string `json:"client"` } -type GetPreviewUrlRequest struct { +type GetPreviewURLRequest struct { FileID int64 `form:"fileID" binding:"required"` Type ente.ObjectType `form:"type" binding:"required"` } -func (g *GetPreviewUrlRequest) Validate() error { +func (g *GetPreviewURLRequest) Validate() error { if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } return nil } + +type PreviewUploadUrlRequest struct { + FileID int64 `form:"fileID" binding:"required"` + Type ente.ObjectType `form:"type" binding:"required"` +} + +func (g *PreviewUploadUrlRequest) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) + } + return nil +} + +// Row represents the data that is stored in the file_data table. +type Row struct { + FileID int64 + UserID int64 + Type ente.ObjectType + // If a file type has multiple objects, then the size is the sum of all the objects. + Size int64 + LatestBucket string + ReplicatedBuckets []string + DeleteFromBuckets []string + InflightReplicas []string + PendingSync bool + IsDeleted bool + SyncLockedTill int64 + CreatedAt int64 + UpdatedAt int64 +} + +// S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket. +func (r *Row) S3FileMetadataObjectKey() string { + if r.Type == ente.DerivedMeta { + return derivedMetaPath(r.FileID, r.UserID) + } + if r.Type == ente.PreviewVideo { + return previewVideoPlaylist(r.FileID, r.UserID) + } + panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) +} + +// GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket. +func (r *Row) GetS3FileObjectKey() string { + if r.Type == ente.PreviewVideo { + return previewVideoPath(r.FileID, r.UserID) + } else if r.Type == ente.PreviewImage { + return previewImagePath(r.FileID, r.UserID) + } + panic(fmt.Sprintf("unsupported object type %s", r.Type)) +} diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go index dcdab16c19..0d70570da3 100644 --- a/server/ente/filedata/putfiledata.go +++ b/server/ente/filedata/putfiledata.go @@ -17,23 +17,27 @@ type PutFileDataRequest struct { Version *int `json:"version,omitempty"` } +func (r PutFileDataRequest) isEncDataPresent() bool { + return r.EncryptedData != nil && r.DecryptionHeader != nil && *r.EncryptedData != "" && *r.DecryptionHeader != "" +} + +func (r PutFileDataRequest) isObjectDataPresent() bool { + return r.ObjectKey != nil && *r.ObjectKey != "" && r.ObjectSize != nil && *r.ObjectSize > 0 +} + func (r PutFileDataRequest) Validate() error { switch r.Type { case ente.PreviewVideo: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - // the video playlist is uploaded as part of encrypted data and decryption header - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") - } - if r.ObjectSize == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") + if !r.isEncDataPresent() || !r.isObjectDataPresent() { + return ente.NewBadRequestWithMessage("object and metadata are required") } case ente.PreviewImage: - if r.ObjectSize == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") + if !r.isObjectDataPresent() || r.isEncDataPresent() { + return ente.NewBadRequestWithMessage("object (only) data is required for preview image") } case ente.DerivedMeta: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta") + if !r.isEncDataPresent() || r.isObjectDataPresent() { + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta") } default: return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 2e15ade325..65efb1d068 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -33,7 +33,7 @@ const DefaultMaxBatchSize = 1000 const DefaultCopyBatchSize = 100 // CreateOrUpdate creates an entry for a file -func (h *FileHandler) CreateOrUpdate(c *gin.Context) { +func (f *FileHandler) CreateOrUpdate(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) var file ente.File if err := c.ShouldBindJSON(&file); err != nil { @@ -48,7 +48,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { if file.ID == 0 { file.OwnerID = userID file.IsDeleted = false - file, err := h.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) + file, err := f.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -56,7 +56,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { c.JSON(http.StatusOK, file) return } - response, err := h.Controller.Update(c, userID, file, enteApp) + response, err := f.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -65,7 +65,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { } // CopyFiles copies files that are owned by another user -func (h *FileHandler) CopyFiles(c *gin.Context) { +func (f *FileHandler) CopyFiles(c *gin.Context) { var req ente.CopyFileSyncRequest if err := c.ShouldBindJSON(&req); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -75,7 +75,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.NewBadRequestWithMessage(fmt.Sprintf("more than %d items", DefaultCopyBatchSize)), "")) return } - response, err := h.FileCopyCtrl.CopyFiles(c, req) + response, err := f.FileCopyCtrl.CopyFiles(c, req) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -84,7 +84,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) { } // Update updates already existing file -func (h *FileHandler) Update(c *gin.Context) { +func (f *FileHandler) Update(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) @@ -98,7 +98,7 @@ func (h *FileHandler) Update(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, "fileID should be >0")) return } - response, err := h.Controller.Update(c, userID, file, enteApp) + response, err := f.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -107,12 +107,12 @@ func (h *FileHandler) Update(c *gin.Context) { } // GetUploadURLs returns a bunch of urls where in the user can upload objects -func (h *FileHandler) GetUploadURLs(c *gin.Context) { +func (f *FileHandler) GetUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false) + urls, err := f.Controller.GetUploadURLs(c, userID, count, enteApp, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -123,12 +123,12 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) { } // GetMultipartUploadURLs returns an array of PartUpload PresignedURLs -func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { +func (f *FileHandler) GetMultipartUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) + urls, err := f.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -139,21 +139,21 @@ func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { } // Get redirects the request to the file location -func (h *FileHandler) Get(c *gin.Context) { +func (f *FileHandler) Get(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetFileURL(c, userID, fileID) + url, err := f.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - h.logBadRedirect(c) + f.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetV2 returns the URL of the file to client -func (h *FileHandler) GetV2(c *gin.Context) { +func (f *FileHandler) GetV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetFileURL(c, userID, fileID) + url, err := f.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -164,21 +164,21 @@ func (h *FileHandler) GetV2(c *gin.Context) { } // GetThumbnail redirects the request to the file's thumbnail location -func (h *FileHandler) GetThumbnail(c *gin.Context) { +func (f *FileHandler) GetThumbnail(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetThumbnailURL(c, userID, fileID) + url, err := f.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - h.logBadRedirect(c) + f.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetThumbnailV2 returns the URL of the thumbnail to the client -func (h *FileHandler) GetThumbnailV2(c *gin.Context) { +func (f *FileHandler) GetThumbnailV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetThumbnailURL(c, userID, fileID) + url, err := f.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -189,7 +189,7 @@ func (h *FileHandler) GetThumbnailV2(c *gin.Context) { } // Trash moves the given files to the trash bin -func (h *FileHandler) Trash(c *gin.Context) { +func (f *FileHandler) Trash(c *gin.Context) { var request ente.TrashRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind")) @@ -201,7 +201,7 @@ func (h *FileHandler) Trash(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) request.OwnerID = userID - err := h.Controller.Trash(c, userID, request) + err := f.Controller.Trash(c, userID, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -210,7 +210,7 @@ func (h *FileHandler) Trash(c *gin.Context) { } // GetSize returns the size of files indicated by fileIDs -func (h *FileHandler) GetSize(c *gin.Context) { +func (f *FileHandler) GetSize(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -227,7 +227,7 @@ func (h *FileHandler) GetSize(c *gin.Context) { return } - size, err := h.Controller.GetSize(userID, request.FileIDs) + size, err := f.Controller.GetSize(userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -238,7 +238,7 @@ func (h *FileHandler) GetSize(c *gin.Context) { } // GetInfo returns the FileInfo of files indicated by fileIDs -func (h *FileHandler) GetInfo(c *gin.Context) { +func (f *FileHandler) GetInfo(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind request")) @@ -246,7 +246,7 @@ func (h *FileHandler) GetInfo(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) - response, err := h.Controller.GetFileInfo(c, userID, request.FileIDs) + response, err := f.Controller.GetFileInfo(c, userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -295,9 +295,9 @@ func shouldRejectRequest(c *gin.Context) (bool, error) { } // GetDuplicates returns the list of files of the same size -func (h *FileHandler) GetDuplicates(c *gin.Context) { +func (f *FileHandler) GetDuplicates(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) - dupes, err := h.Controller.GetDuplicates(userID) + dupes, err := f.Controller.GetDuplicates(userID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -308,10 +308,10 @@ func (h *FileHandler) GetDuplicates(c *gin.Context) { } // GetLargeThumbnail returns the list of files whose thumbnail size is larger than threshold size -func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { +func (f *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) threshold, _ := strconv.ParseInt(c.Query("threshold"), 10, 64) - largeThumbnailFiles, err := h.Controller.GetLargeThumbnailFiles(userID, threshold) + largeThumbnailFiles, err := f.Controller.GetLargeThumbnailFiles(userID, threshold) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -322,7 +322,7 @@ func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { } // UpdateMagicMetadata updates magic metadata for a list of files. -func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { +func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -332,7 +332,7 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBatchSizeTooLarge, "")) return } - err := h.Controller.UpdateMagicMetadata(c, request, false) + err := f.Controller.UpdateMagicMetadata(c, request, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -341,13 +341,13 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { } // UpdatePublicMagicMetadata updates public magic metadata for a list of files. -func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { +func (f *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := h.Controller.UpdateMagicMetadata(c, request, true) + err := f.Controller.UpdateMagicMetadata(c, request, true) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -356,7 +356,7 @@ func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { } // UpdateThumbnail updates thumbnail of a file -func (h *FileHandler) UpdateThumbnail(c *gin.Context) { +func (f *FileHandler) UpdateThumbnail(c *gin.Context) { enteApp := auth.GetApp(c) var request ente.UpdateThumbnailRequest @@ -364,7 +364,7 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := h.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) + err := f.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -372,8 +372,8 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) { c.Status(http.StatusOK) } -func (h *FileHandler) GetTotalFileCount(c *gin.Context) { - count, err := h.Controller.GetTotalFileCount() +func (f *FileHandler) GetTotalFileCount(c *gin.Context) { + count, err := f.Controller.GetTotalFileCount() if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -390,7 +390,7 @@ func getUserAndFileIDs(c *gin.Context) (int64, int64) { } // logBadRedirect will log the request id if we are redirecting to another url with the auth-token in header -func (h *FileHandler) logBadRedirect(c *gin.Context) { +func (f *FileHandler) logBadRedirect(c *gin.Context) { if len(c.GetHeader("X-Auth-Token")) != 0 && os.Getenv("ENVIRONMENT") != "" { log.WithField("req_id", requestid.Get(c)).Error("critical: sending token to another service") } diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go index 50015c3d4d..d45a3392b8 100644 --- a/server/pkg/api/file_data.go +++ b/server/pkg/api/file_data.go @@ -4,12 +4,10 @@ import ( "fmt" "github.com/ente-io/museum/ente" fileData "github.com/ente-io/museum/ente/filedata" - "github.com/ente-io/museum/pkg/utils/auth" "github.com/ente-io/museum/pkg/utils/handler" "github.com/ente-io/stacktrace" "github.com/gin-gonic/gin" "net/http" - "strconv" ) func (f *FileHandler) PutFileData(ctx *gin.Context) { @@ -50,20 +48,29 @@ func (f *FileHandler) GetFilesData(ctx *gin.Context) { ctx.JSON(http.StatusOK, resp) } -func (h *FileHandler) GetVideoUploadURL(c *gin.Context) { - enteApp := auth.GetApp(c) - userID, fileID := getUserAndFileIDs(c) - urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) +func (f *FileHandler) GetFileData(ctx *gin.Context) { + var req fileData.GetFileData + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) return } - c.JSON(http.StatusOK, urls) + resp, err := f.FileDataCtrl.GetFileData(ctx, req) + if err != nil { + handler.Error(ctx, err) + return + } + ctx.JSON(http.StatusOK, gin.H{ + "data": resp, + }) } -func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { - userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetPreviewUrl(c, userID, fileID) +func (f *FileHandler) GetPreviewUploadURL(c *gin.Context) { + var request fileData.PreviewUploadUrlRequest + if err := c.ShouldBindJSON(&request); err != nil { + handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + return + } + url, err := f.FileDataCtrl.PreviewUploadURL(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -73,27 +80,18 @@ func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { }) } -func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { - var request ente.InsertOrUpdateEmbeddingRequest +func (f *FileHandler) GetPreviewURL(c *gin.Context) { + var request fileData.GetPreviewURLRequest if err := c.ShouldBindJSON(&request); err != nil { - handler.Error(c, - stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) return } - err := h.Controller.ReportVideoPreview(c, request) + url, err := f.FileDataCtrl.GetPreviewUrl(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - c.Status(http.StatusOK) -} - -func (h *FileHandler) GetVideoPlaylist(c *gin.Context) { - fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64) - response, err := h.Controller.GetPlaylist(c, fileID) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) - return - } - c.JSON(http.StatusOK, response) + c.JSON(http.StatusOK, gin.H{ + "url": url, + }) } diff --git a/server/pkg/controller/file_preview.go b/server/pkg/controller/file_preview.go index d998267319..5996bc4f70 100644 --- a/server/pkg/controller/file_preview.go +++ b/server/pkg/controller/file_preview.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/ente-io/museum/ente" @@ -22,53 +21,6 @@ const ( _model = "hls_video" ) -// GetUploadURLs returns a bunch of presigned URLs for uploading files -func (c *FileController) GetVideoUploadUrl(ctx context.Context, userID int64, fileID int64, app ente.App) (*ente.UploadURL, error) { - err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app) - if err != nil { - return nil, stacktrace.Propagate(err, "") - } - s3Client := c.S3Config.GetDerivedStorageS3Client() - dc := c.S3Config.GetDerivedStorageDataCenter() - bucket := c.S3Config.GetDerivedStorageBucket() - objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/" + _model - url, err := c.getObjectURL(s3Client, dc, bucket, objectKey) - if err != nil { - return nil, stacktrace.Propagate(err, "") - } - log.Infof("Got upload URL for %s", objectKey) - return &url, nil -} - -func (c *FileController) GetPreviewUrl(ctx context.Context, userID int64, fileID int64) (string, error) { - err := c.verifyFileAccess(userID, fileID) - if err != nil { - return "", err - } - objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video" - // check if playlist exists - err = c.checkObjectExists(ctx, objectKey+"_playlist.m3u8", c.S3Config.GetDerivedStorageDataCenter()) - if err != nil { - return "", stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) - } - s3Client := c.S3Config.GetDerivedStorageS3Client() - r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ - Bucket: c.S3Config.GetDerivedStorageBucket(), - Key: &objectKey, - }) - return r.Presign(PreSignedRequestValidityDuration) -} - -func (c *FileController) GetPlaylist(ctx *gin.Context, fileID int64) (ente.EmbeddingObject, error) { - objectKey := strconv.FormatInt(auth.GetUserID(ctx.Request.Header), 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video_playlist.m3u8" - // check if object exists - err := c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) - if err != nil { - return ente.EmbeddingObject{}, stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) - } - return c.downloadObject(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) -} - func (c *FileController) ReportVideoPreview(ctx *gin.Context, req ente.InsertOrUpdateEmbeddingRequest) error { userID := auth.GetUserID(ctx.Request.Header) if strings.Compare(req.Model, "hls_video") != 0 { @@ -128,26 +80,6 @@ func (c *FileController) uploadObject(obj ente.EmbeddingObject, key string, dc s return len(embeddingObj), nil } -func (c *FileController) downloadObject(ctx context.Context, objectKey string, dc string) (ente.EmbeddingObject, error) { - var obj ente.EmbeddingObject - buff := &aws.WriteAtBuffer{} - bucket := c.S3Config.GetBucket(dc) - s3Client := c.S3Config.GetS3Client(dc) - downloader := s3manager.NewDownloaderWithClient(&s3Client) - _, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{ - Bucket: bucket, - Key: &objectKey, - }) - if err != nil { - return obj, err - } - err = json.Unmarshal(buff.Bytes(), &obj) - if err != nil { - return obj, stacktrace.Propagate(err, "unmarshal failed") - } - return obj, nil -} - func (c *FileController) checkObjectExists(ctx context.Context, objectKey string, dc string) error { s3Client := c.S3Config.GetS3Client(dc) _, err := s3Client.HeadObject(&s3.HeadObjectInput{ diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index afd0a73a6f..b01cec9c48 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -83,10 +83,13 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR return stacktrace.Propagate(err, "validation failed") } userID := auth.GetUserID(ctx.Request.Header) - err := c._validateInsertPermission(ctx, req.FileID, userID) + err := c._validatePermission(ctx, req.FileID, userID) if err != nil { return stacktrace.Propagate(err, "") } + if req.Type != ente.DerivedMeta { + return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "") + } fileOwnerID := userID objectKey := req.S3FileMetadataObjectKey(fileOwnerID) obj := fileData.S3FileMetadata{ @@ -115,6 +118,32 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR return nil } +func (c *Controller) GetFileData(ctx *gin.Context, req fileData.GetFileData) (*fileData.Entity, error) { + if err := req.Validate(); err != nil { + return nil, stacktrace.Propagate(err, "validation failed") + } + if err := c._validatePermission(ctx, req.FileID, auth.GetUserID(ctx.Request.Header)); err != nil { + return nil, stacktrace.Propagate(err, "") + } + doRows, err := c.Repo.GetFilesData(ctx, req.Type, []int64{req.FileID}) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if len(doRows) == 0 || doRows[0].IsDeleted { + return nil, stacktrace.Propagate(ente.ErrNotFound, "") + } + s3MetaObject, err := c.fetchS3FileMetadata(context.Background(), doRows[0], doRows[0].LatestBucket) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &fileData.Entity{ + FileID: doRows[0].FileID, + Type: doRows[0].Type, + EncryptedData: s3MetaObject.EncryptedData, + DecryptionHeader: s3MetaObject.DecryptionHeader, + }, nil +} + func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) (*fileData.GetFilesDataResponse, error) { userID := auth.GetUserID(ctx.Request.Header) if err := c._validateGetFilesData(ctx, userID, req); err != nil { @@ -254,7 +283,7 @@ func (c *Controller) _validateGetFilesData(ctx *gin.Context, userID int64, req f return nil } -func (c *Controller) _validateInsertPermission(ctx *gin.Context, fileID int64, actorID int64) error { +func (c *Controller) _validatePermission(ctx *gin.Context, fileID int64, actorID int64) error { err := c.AccessCtrl.VerifyFileOwnership(ctx, &access.VerifyFileOwnershipParams{ ActorUserId: actorID, FileIDs: []int64{fileID}, diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go new file mode 100644 index 0000000000..4d6b0113ba --- /dev/null +++ b/server/pkg/controller/filedata/preview_files.go @@ -0,0 +1,54 @@ +package filedata + +import ( + "fmt" + "github.com/ente-io/museum/ente" + "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/utils/auth" + "github.com/ente-io/stacktrace" + "github.com/gin-gonic/gin" +) + +func (c *Controller) GetPreviewUrl(ctx *gin.Context, request filedata.GetPreviewURLRequest) (*string, error) { + if err := request.Validate(); err != nil { + return nil, err + } + actorUser := auth.GetUserID(ctx.Request.Header) + if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil { + return nil, err + } + data, err := c.Repo.GetFilesData(ctx, request.Type, []int64{request.FileID}) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if len(data) == 0 || data[0].IsDeleted { + return nil, stacktrace.Propagate(ente.ErrNotFound, "") + } + enteUrl, err := c.signedUrlGet(data[0].LatestBucket, data[0].GetS3FileObjectKey()) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &enteUrl.URL, nil +} + +func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewUploadUrlRequest) (*string, error) { + if err := request.Validate(); err != nil { + return nil, err + } + actorUser := auth.GetUserID(ctx.Request.Header) + if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil { + return nil, err + } + fileOwnerID, err := c.FileRepo.GetOwnerID(request.FileID) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + // note: instead of the final url, give a temp url for upload purpose. + uploadUrl := fmt.Sprintf("%s_temp_upload", filedata.PreviewUrl(request.FileID, fileOwnerID, request.Type)) + bucketID := c.S3Config.GetBucketID(request.Type) + enteUrl, err := c.getUploadURL(bucketID, uploadUrl) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &enteUrl.URL, nil +} diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go index f8a55052ec..415e28cb28 100644 --- a/server/pkg/controller/filedata/s3.go +++ b/server/pkg/controller/filedata/s3.go @@ -7,11 +7,51 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/ente-io/museum/ente" fileData "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/stacktrace" log "github.com/sirupsen/logrus" + stime "time" ) +const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour + +func (c *Controller) getUploadURL(dc string, objectKey string) (*ente.UploadURL, error) { + s3Client := c.S3Config.GetS3Client(dc) + r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: c.S3Config.GetBucket(dc), + Key: &objectKey, + }) + url, err := r.Presign(PreSignedRequestValidityDuration) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + err = c.ObjectCleanupController.AddTempObjectKey(objectKey, dc) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &ente.UploadURL{ + ObjectKey: objectKey, + URL: url, + }, nil +} + +func (c *Controller) signedUrlGet(dc string, objectKey string) (*ente.UploadURL, error) { + s3Client := c.S3Config.GetS3Client(dc) + r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: c.S3Config.GetBucket(dc), + Key: &objectKey, + }) + url, err := r.Presign(PreSignedRequestValidityDuration) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &ente.UploadURL{ObjectKey: objectKey, URL: url}, nil +} + func (c *Controller) downloadObject(ctx context.Context, objectKey string, dc string) (fileData.S3FileMetadata, error) { var obj fileData.S3FileMetadata buff := &aws.WriteAtBuffer{}