TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[server] 1/n Support for persisting preview video

Browse Source
This commit is contained in:
Neeraj Gupta
2024-08-09 15:50:29 +05:30
parent 7834662340
commit cd2fde2c2e
9 changed files with 277 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/cmd/museum/main.go
View File

@@ -413,12 +413,11 @@ func main() {
privateAPI.GET("/files/preview/:fileID", fileHandler.GetThumbnail)
privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail)
privateAPI.GET("/files/file-data/playlist/:fileID", fileHandler.GetVideoPlaylist)
privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList)
privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL)
privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl)
privateAPI.PUT("/files/data/", fileHandler.PutFileData)
privateAPI.POST("files/fetch-data/", fileHandler.GetFilesData)
privateAPI.POST("files/data/fetch", fileHandler.GetFilesData)
privateAPI.GET("files/data/fetch", fileHandler.GetFileData)
privateAPI.GET("/files/data/preview-upload-url/", fileHandler.GetPreviewUploadURL)
privateAPI.GET("/files/data/preview/", fileHandler.GetPreviewURL)
privateAPI.POST("/files", fileHandler.CreateOrUpdate)
privateAPI.POST("/files/copy", fileHandler.CopyFiles)

100
server/ente/filedata/filedata.go
View File

@@ -5,9 +5,16 @@ import (
"github.com/ente-io/museum/ente"
)
type Entity struct {
FileID int64 `json:"fileID"`
Type ente.ObjectType `json:"type"`
EncryptedData string `json:"encryptedData"`
DecryptionHeader string `json:"decryptionHeader"`
}
// GetFilesData should only be used for getting the preview video playlist and derived metadata.
type GetFilesData struct {
FileIDs []int64 `form:"fileIDs" binding:"required"`
FileIDs []int64 `json:"fileIDs" binding:"required"`
Type ente.ObjectType `json:"type" binding:"required"`
}
@@ -24,38 +31,16 @@ func (g *GetFilesData) Validate() error {
return nil
}
type Entity struct {
FileID int64 `json:"fileID"`
Type ente.ObjectType `json:"type"`
EncryptedData string `json:"encryptedData"`
DecryptionHeader string `json:"decryptionHeader"`
type GetFileData struct {
FileID int64 `form:"fileID" binding:"required"`
Type ente.ObjectType `form:"type" binding:"required"`
}
// Row represents the data that is stored in the file_data table.
type Row struct {
FileID int64
UserID int64
Type ente.ObjectType
Size int64
LatestBucket string
ReplicatedBuckets []string
DeleteFromBuckets []string
InflightReplicas []string
PendingSync bool
IsDeleted bool
SyncLockedTill int64
CreatedAt int64
UpdatedAt int64
}
func (r Row) S3FileMetadataObjectKey() string {
if r.Type == ente.DerivedMeta {
return derivedMetaPath(r.FileID, r.UserID)
func (g *GetFileData) Validate() error {
if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta {
return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type))
}
if r.Type == ente.PreviewVideo {
return previewVideoPlaylist(r.FileID, r.UserID)
}
panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
return nil
}
type GetFilesDataResponse struct {
@@ -73,14 +58,65 @@ type S3FileMetadata struct {
Client string `json:"client"`
}
type GetPreviewUrlRequest struct {
type GetPreviewURLRequest struct {
FileID int64 `form:"fileID" binding:"required"`
Type ente.ObjectType `form:"type" binding:"required"`
}
func (g *GetPreviewUrlRequest) Validate() error {
func (g *GetPreviewURLRequest) Validate() error {
if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage {
return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type))
}
return nil
}
type PreviewUploadUrlRequest struct {
FileID int64 `form:"fileID" binding:"required"`
Type ente.ObjectType `form:"type" binding:"required"`
}
func (g *PreviewUploadUrlRequest) Validate() error {
if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage {
return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type))
}
return nil
}
// Row represents the data that is stored in the file_data table.
type Row struct {
FileID int64
UserID int64
Type ente.ObjectType
// If a file type has multiple objects, then the size is the sum of all the objects.
Size int64
LatestBucket string
ReplicatedBuckets []string
DeleteFromBuckets []string
InflightReplicas []string
PendingSync bool
IsDeleted bool
SyncLockedTill int64
CreatedAt int64
UpdatedAt int64
}
// S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket.
func (r *Row) S3FileMetadataObjectKey() string {
if r.Type == ente.DerivedMeta {
return derivedMetaPath(r.FileID, r.UserID)
}
if r.Type == ente.PreviewVideo {
return previewVideoPlaylist(r.FileID, r.UserID)
}
panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
}
// GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket.
func (r *Row) GetS3FileObjectKey() string {
if r.Type == ente.PreviewVideo {
return previewVideoPath(r.FileID, r.UserID)
} else if r.Type == ente.PreviewImage {
return previewImagePath(r.FileID, r.UserID)
}
panic(fmt.Sprintf("unsupported object type %s", r.Type))
}

24
server/ente/filedata/putfiledata.go
View File

@@ -17,23 +17,27 @@ type PutFileDataRequest struct {
Version *int `json:"version,omitempty"`
}
func (r PutFileDataRequest) isEncDataPresent() bool {
return r.EncryptedData != nil && r.DecryptionHeader != nil && *r.EncryptedData != "" && *r.DecryptionHeader != ""
}
func (r PutFileDataRequest) isObjectDataPresent() bool {
return r.ObjectKey != nil && *r.ObjectKey != "" && r.ObjectSize != nil && *r.ObjectSize > 0
}
func (r PutFileDataRequest) Validate() error {
switch r.Type {
case ente.PreviewVideo:
if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" {
// the video playlist is uploaded as part of encrypted data and decryption header
return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video")
}
if r.ObjectSize == nil || r.ObjectKey == nil {
return ente.NewBadRequestWithMessage("size and objectKey are required for preview video")
if !r.isEncDataPresent() || !r.isObjectDataPresent() {
return ente.NewBadRequestWithMessage("object and metadata are required")
}
case ente.PreviewImage:
if r.ObjectSize == nil || r.ObjectKey == nil {
return ente.NewBadRequestWithMessage("size and objectKey are required for preview image")
if !r.isObjectDataPresent() || r.isEncDataPresent() {
return ente.NewBadRequestWithMessage("object (only) data is required for preview image")
}
case ente.DerivedMeta:
if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" {
return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta")
if !r.isEncDataPresent() || r.isObjectDataPresent() {
return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta")
}
default:
return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type))

80
server/pkg/api/file.go
View File

@@ -33,7 +33,7 @@ const DefaultMaxBatchSize = 1000
const DefaultCopyBatchSize = 100
// CreateOrUpdate creates an entry for a file
func (h *FileHandler) CreateOrUpdate(c *gin.Context) {
func (f *FileHandler) CreateOrUpdate(c *gin.Context) {
userID := auth.GetUserID(c.Request.Header)
var file ente.File
if err := c.ShouldBindJSON(&file); err != nil {
@@ -48,7 +48,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) {
if file.ID == 0 {
file.OwnerID = userID
file.IsDeleted = false
file, err := h.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp)
file, err := f.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -56,7 +56,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) {
c.JSON(http.StatusOK, file)
return
}
response, err := h.Controller.Update(c, userID, file, enteApp)
response, err := f.Controller.Update(c, userID, file, enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -65,7 +65,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) {
}
// CopyFiles copies files that are owned by another user
func (h *FileHandler) CopyFiles(c *gin.Context) {
func (f *FileHandler) CopyFiles(c *gin.Context) {
var req ente.CopyFileSyncRequest
if err := c.ShouldBindJSON(&req); err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
@@ -75,7 +75,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(ente.NewBadRequestWithMessage(fmt.Sprintf("more than %d items", DefaultCopyBatchSize)), ""))
return
}
response, err := h.FileCopyCtrl.CopyFiles(c, req)
response, err := f.FileCopyCtrl.CopyFiles(c, req)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -84,7 +84,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) {
}
// Update updates already existing file
func (h *FileHandler) Update(c *gin.Context) {
func (f *FileHandler) Update(c *gin.Context) {
enteApp := auth.GetApp(c)
userID := auth.GetUserID(c.Request.Header)
@@ -98,7 +98,7 @@ func (h *FileHandler) Update(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, "fileID should be >0"))
return
}
response, err := h.Controller.Update(c, userID, file, enteApp)
response, err := f.Controller.Update(c, userID, file, enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -107,12 +107,12 @@ func (h *FileHandler) Update(c *gin.Context) {
}
// GetUploadURLs returns a bunch of urls where in the user can upload objects
func (h *FileHandler) GetUploadURLs(c *gin.Context) {
func (f *FileHandler) GetUploadURLs(c *gin.Context) {
enteApp := auth.GetApp(c)
userID := auth.GetUserID(c.Request.Header)
count, _ := strconv.Atoi(c.Query("count"))
urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false)
urls, err := f.Controller.GetUploadURLs(c, userID, count, enteApp, false)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -123,12 +123,12 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) {
}
// GetMultipartUploadURLs returns an array of PartUpload PresignedURLs
func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) {
func (f *FileHandler) GetMultipartUploadURLs(c *gin.Context) {
enteApp := auth.GetApp(c)
userID := auth.GetUserID(c.Request.Header)
count, _ := strconv.Atoi(c.Query("count"))
urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp)
urls, err := f.Controller.GetMultipartUploadURLs(c, userID, count, enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -139,21 +139,21 @@ func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) {
}
// Get redirects the request to the file location
func (h *FileHandler) Get(c *gin.Context) {
func (f *FileHandler) Get(c *gin.Context) {
userID, fileID := getUserAndFileIDs(c)
url, err := h.Controller.GetFileURL(c, userID, fileID)
url, err := f.Controller.GetFileURL(c, userID, fileID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
h.logBadRedirect(c)
f.logBadRedirect(c)
c.Redirect(http.StatusTemporaryRedirect, url)
}
// GetV2 returns the URL of the file to client
func (h *FileHandler) GetV2(c *gin.Context) {
func (f *FileHandler) GetV2(c *gin.Context) {
userID, fileID := getUserAndFileIDs(c)
url, err := h.Controller.GetFileURL(c, userID, fileID)
url, err := f.Controller.GetFileURL(c, userID, fileID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -164,21 +164,21 @@ func (h *FileHandler) GetV2(c *gin.Context) {
}
// GetThumbnail redirects the request to the file's thumbnail location
func (h *FileHandler) GetThumbnail(c *gin.Context) {
func (f *FileHandler) GetThumbnail(c *gin.Context) {
userID, fileID := getUserAndFileIDs(c)
url, err := h.Controller.GetThumbnailURL(c, userID, fileID)
url, err := f.Controller.GetThumbnailURL(c, userID, fileID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
h.logBadRedirect(c)
f.logBadRedirect(c)
c.Redirect(http.StatusTemporaryRedirect, url)
}
// GetThumbnailV2 returns the URL of the thumbnail to the client
func (h *FileHandler) GetThumbnailV2(c *gin.Context) {
func (f *FileHandler) GetThumbnailV2(c *gin.Context) {
userID, fileID := getUserAndFileIDs(c)
url, err := h.Controller.GetThumbnailURL(c, userID, fileID)
url, err := f.Controller.GetThumbnailURL(c, userID, fileID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -189,7 +189,7 @@ func (h *FileHandler) GetThumbnailV2(c *gin.Context) {
}
// Trash moves the given files to the trash bin
func (h *FileHandler) Trash(c *gin.Context) {
func (f *FileHandler) Trash(c *gin.Context) {
var request ente.TrashRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(err, "failed to bind"))
@@ -201,7 +201,7 @@ func (h *FileHandler) Trash(c *gin.Context) {
}
userID := auth.GetUserID(c.Request.Header)
request.OwnerID = userID
err := h.Controller.Trash(c, userID, request)
err := f.Controller.Trash(c, userID, request)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
} else {
@@ -210,7 +210,7 @@ func (h *FileHandler) Trash(c *gin.Context) {
}
// GetSize returns the size of files indicated by fileIDs
func (h *FileHandler) GetSize(c *gin.Context) {
func (f *FileHandler) GetSize(c *gin.Context) {
var request ente.FileIDsRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
@@ -227,7 +227,7 @@ func (h *FileHandler) GetSize(c *gin.Context) {
return
}
size, err := h.Controller.GetSize(userID, request.FileIDs)
size, err := f.Controller.GetSize(userID, request.FileIDs)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
} else {
@@ -238,7 +238,7 @@ func (h *FileHandler) GetSize(c *gin.Context) {
}
// GetInfo returns the FileInfo of files indicated by fileIDs
func (h *FileHandler) GetInfo(c *gin.Context) {
func (f *FileHandler) GetInfo(c *gin.Context) {
var request ente.FileIDsRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(err, "failed to bind request"))
@@ -246,7 +246,7 @@ func (h *FileHandler) GetInfo(c *gin.Context) {
}
userID := auth.GetUserID(c.Request.Header)
response, err := h.Controller.GetFileInfo(c, userID, request.FileIDs)
response, err := f.Controller.GetFileInfo(c, userID, request.FileIDs)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
} else {
@@ -295,9 +295,9 @@ func shouldRejectRequest(c *gin.Context) (bool, error) {
}
// GetDuplicates returns the list of files of the same size
func (h *FileHandler) GetDuplicates(c *gin.Context) {
func (f *FileHandler) GetDuplicates(c *gin.Context) {
userID := auth.GetUserID(c.Request.Header)
dupes, err := h.Controller.GetDuplicates(userID)
dupes, err := f.Controller.GetDuplicates(userID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -308,10 +308,10 @@ func (h *FileHandler) GetDuplicates(c *gin.Context) {
}
// GetLargeThumbnail returns the list of files whose thumbnail size is larger than threshold size
func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) {
func (f *FileHandler) GetLargeThumbnailFiles(c *gin.Context) {
userID := auth.GetUserID(c.Request.Header)
threshold, _ := strconv.ParseInt(c.Query("threshold"), 10, 64)
largeThumbnailFiles, err := h.Controller.GetLargeThumbnailFiles(userID, threshold)
largeThumbnailFiles, err := f.Controller.GetLargeThumbnailFiles(userID, threshold)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -322,7 +322,7 @@ func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) {
}
// UpdateMagicMetadata updates magic metadata for a list of files.
func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) {
func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) {
var request ente.UpdateMultipleMagicMetadataRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
@@ -332,7 +332,7 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(ente.ErrBatchSizeTooLarge, ""))
return
}
err := h.Controller.UpdateMagicMetadata(c, request, false)
err := f.Controller.UpdateMagicMetadata(c, request, false)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -341,13 +341,13 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) {
}
// UpdatePublicMagicMetadata updates public magic metadata for a list of files.
func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) {
func (f *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) {
var request ente.UpdateMultipleMagicMetadataRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
err := h.Controller.UpdateMagicMetadata(c, request, true)
err := f.Controller.UpdateMagicMetadata(c, request, true)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -356,7 +356,7 @@ func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) {
}
// UpdateThumbnail updates thumbnail of a file
func (h *FileHandler) UpdateThumbnail(c *gin.Context) {
func (f *FileHandler) UpdateThumbnail(c *gin.Context) {
enteApp := auth.GetApp(c)
var request ente.UpdateThumbnailRequest
@@ -364,7 +364,7 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
err := h.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp)
err := f.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -372,8 +372,8 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) {
c.Status(http.StatusOK)
}
func (h *FileHandler) GetTotalFileCount(c *gin.Context) {
count, err := h.Controller.GetTotalFileCount()
func (f *FileHandler) GetTotalFileCount(c *gin.Context) {
count, err := f.Controller.GetTotalFileCount()
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -390,7 +390,7 @@ func getUserAndFileIDs(c *gin.Context) (int64, int64) {
}
// logBadRedirect will log the request id if we are redirecting to another url with the auth-token in header
func (h *FileHandler) logBadRedirect(c *gin.Context) {
func (f *FileHandler) logBadRedirect(c *gin.Context) {
if len(c.GetHeader("X-Auth-Token")) != 0 && os.Getenv("ENVIRONMENT") != "" {
log.WithField("req_id", requestid.Get(c)).Error("critical: sending token to another service")
}

54
server/pkg/api/file_data.go
View File

@@ -4,12 +4,10 @@ import (
"fmt"
"github.com/ente-io/museum/ente"
fileData "github.com/ente-io/museum/ente/filedata"
"github.com/ente-io/museum/pkg/utils/auth"
"github.com/ente-io/museum/pkg/utils/handler"
"github.com/ente-io/stacktrace"
"github.com/gin-gonic/gin"
"net/http"
"strconv"
)
func (f *FileHandler) PutFileData(ctx *gin.Context) {
@@ -50,20 +48,29 @@ func (f *FileHandler) GetFilesData(ctx *gin.Context) {
ctx.JSON(http.StatusOK, resp)
}
func (h *FileHandler) GetVideoUploadURL(c *gin.Context) {
enteApp := auth.GetApp(c)
userID, fileID := getUserAndFileIDs(c)
urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
func (f *FileHandler) GetFileData(ctx *gin.Context) {
var req fileData.GetFileData
if err := ctx.ShouldBindJSON(&req); err != nil {
ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error()))
return
}
c.JSON(http.StatusOK, urls)
resp, err := f.FileDataCtrl.GetFileData(ctx, req)
if err != nil {
handler.Error(ctx, err)
return
}
ctx.JSON(http.StatusOK, gin.H{
"data": resp,
})
}
func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) {
userID, fileID := getUserAndFileIDs(c)
url, err := h.Controller.GetPreviewUrl(c, userID, fileID)
func (f *FileHandler) GetPreviewUploadURL(c *gin.Context) {
var request fileData.PreviewUploadUrlRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err)))
return
}
url, err := f.FileDataCtrl.PreviewUploadURL(c, request)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -73,27 +80,18 @@ func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) {
})
}
func (h *FileHandler) ReportVideoPlayList(c *gin.Context) {
var request ente.InsertOrUpdateEmbeddingRequest
func (f *FileHandler) GetPreviewURL(c *gin.Context) {
var request fileData.GetPreviewURLRequest
if err := c.ShouldBindJSON(&request); err != nil {
handler.Error(c,
stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err)))
handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err)))
return
}
err := h.Controller.ReportVideoPreview(c, request)
url, err := f.FileDataCtrl.GetPreviewUrl(c, request)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
c.Status(http.StatusOK)
}
func (h *FileHandler) GetVideoPlaylist(c *gin.Context) {
fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64)
response, err := h.Controller.GetPlaylist(c, fileID)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
c.JSON(http.StatusOK, response)
c.JSON(http.StatusOK, gin.H{
"url": url,
})
}

68
server/pkg/controller/file_preview.go
View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/ente-io/museum/ente"
@@ -22,53 +21,6 @@ const (
_model = "hls_video"
)
// GetUploadURLs returns a bunch of presigned URLs for uploading files
func (c *FileController) GetVideoUploadUrl(ctx context.Context, userID int64, fileID int64, app ente.App) (*ente.UploadURL, error) {
err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
s3Client := c.S3Config.GetDerivedStorageS3Client()
dc := c.S3Config.GetDerivedStorageDataCenter()
bucket := c.S3Config.GetDerivedStorageBucket()
objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/" + _model
url, err := c.getObjectURL(s3Client, dc, bucket, objectKey)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
log.Infof("Got upload URL for %s", objectKey)
return &url, nil
}
func (c *FileController) GetPreviewUrl(ctx context.Context, userID int64, fileID int64) (string, error) {
err := c.verifyFileAccess(userID, fileID)
if err != nil {
return "", err
}
objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video"
// check if playlist exists
err = c.checkObjectExists(ctx, objectKey+"_playlist.m3u8", c.S3Config.GetDerivedStorageDataCenter())
if err != nil {
return "", stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey))
}
s3Client := c.S3Config.GetDerivedStorageS3Client()
r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{
Bucket: c.S3Config.GetDerivedStorageBucket(),
Key: &objectKey,
})
return r.Presign(PreSignedRequestValidityDuration)
}
func (c *FileController) GetPlaylist(ctx *gin.Context, fileID int64) (ente.EmbeddingObject, error) {
objectKey := strconv.FormatInt(auth.GetUserID(ctx.Request.Header), 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video_playlist.m3u8"
// check if object exists
err := c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter())
if err != nil {
return ente.EmbeddingObject{}, stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey))
}
return c.downloadObject(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter())
}
func (c *FileController) ReportVideoPreview(ctx *gin.Context, req ente.InsertOrUpdateEmbeddingRequest) error {
userID := auth.GetUserID(ctx.Request.Header)
if strings.Compare(req.Model, "hls_video") != 0 {
@@ -128,26 +80,6 @@ func (c *FileController) uploadObject(obj ente.EmbeddingObject, key string, dc s
return len(embeddingObj), nil
}
func (c *FileController) downloadObject(ctx context.Context, objectKey string, dc string) (ente.EmbeddingObject, error) {
var obj ente.EmbeddingObject
buff := &aws.WriteAtBuffer{}
bucket := c.S3Config.GetBucket(dc)
s3Client := c.S3Config.GetS3Client(dc)
downloader := s3manager.NewDownloaderWithClient(&s3Client)
_, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{
Bucket: bucket,
Key: &objectKey,
})
if err != nil {
return obj, err
}
err = json.Unmarshal(buff.Bytes(), &obj)
if err != nil {
return obj, stacktrace.Propagate(err, "unmarshal failed")
}
return obj, nil
}
func (c *FileController) checkObjectExists(ctx context.Context, objectKey string, dc string) error {
s3Client := c.S3Config.GetS3Client(dc)
_, err := s3Client.HeadObject(&s3.HeadObjectInput{

33
server/pkg/controller/filedata/controller.go
View File

@@ -83,10 +83,13 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR
return stacktrace.Propagate(err, "validation failed")
}
userID := auth.GetUserID(ctx.Request.Header)
err := c._validateInsertPermission(ctx, req.FileID, userID)
err := c._validatePermission(ctx, req.FileID, userID)
if err != nil {
return stacktrace.Propagate(err, "")
}
if req.Type != ente.DerivedMeta {
return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "")
}
fileOwnerID := userID
objectKey := req.S3FileMetadataObjectKey(fileOwnerID)
obj := fileData.S3FileMetadata{
@@ -115,6 +118,32 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR
return nil
}
func (c *Controller) GetFileData(ctx *gin.Context, req fileData.GetFileData) (*fileData.Entity, error) {
if err := req.Validate(); err != nil {
return nil, stacktrace.Propagate(err, "validation failed")
}
if err := c._validatePermission(ctx, req.FileID, auth.GetUserID(ctx.Request.Header)); err != nil {
return nil, stacktrace.Propagate(err, "")
}
doRows, err := c.Repo.GetFilesData(ctx, req.Type, []int64{req.FileID})
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
if len(doRows) == 0 || doRows[0].IsDeleted {
return nil, stacktrace.Propagate(ente.ErrNotFound, "")
}
s3MetaObject, err := c.fetchS3FileMetadata(context.Background(), doRows[0], doRows[0].LatestBucket)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &fileData.Entity{
FileID: doRows[0].FileID,
Type: doRows[0].Type,
EncryptedData: s3MetaObject.EncryptedData,
DecryptionHeader: s3MetaObject.DecryptionHeader,
}, nil
}
func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) (*fileData.GetFilesDataResponse, error) {
userID := auth.GetUserID(ctx.Request.Header)
if err := c._validateGetFilesData(ctx, userID, req); err != nil {
@@ -254,7 +283,7 @@ func (c *Controller) _validateGetFilesData(ctx *gin.Context, userID int64, req f
return nil
}
func (c *Controller) _validateInsertPermission(ctx *gin.Context, fileID int64, actorID int64) error {
func (c *Controller) _validatePermission(ctx *gin.Context, fileID int64, actorID int64) error {
err := c.AccessCtrl.VerifyFileOwnership(ctx, &access.VerifyFileOwnershipParams{
ActorUserId: actorID,
FileIDs: []int64{fileID},

54
server/pkg/controller/filedata/preview_files.go Normal file
View File

@@ -0,0 +1,54 @@
package filedata
import (
"fmt"
"github.com/ente-io/museum/ente"
"github.com/ente-io/museum/ente/filedata"
"github.com/ente-io/museum/pkg/utils/auth"
"github.com/ente-io/stacktrace"
"github.com/gin-gonic/gin"
)
func (c *Controller) GetPreviewUrl(ctx *gin.Context, request filedata.GetPreviewURLRequest) (*string, error) {
if err := request.Validate(); err != nil {
return nil, err
}
actorUser := auth.GetUserID(ctx.Request.Header)
if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil {
return nil, err
}
data, err := c.Repo.GetFilesData(ctx, request.Type, []int64{request.FileID})
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
if len(data) == 0 || data[0].IsDeleted {
return nil, stacktrace.Propagate(ente.ErrNotFound, "")
}
enteUrl, err := c.signedUrlGet(data[0].LatestBucket, data[0].GetS3FileObjectKey())
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &enteUrl.URL, nil
}
func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewUploadUrlRequest) (*string, error) {
if err := request.Validate(); err != nil {
return nil, err
}
actorUser := auth.GetUserID(ctx.Request.Header)
if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil {
return nil, err
}
fileOwnerID, err := c.FileRepo.GetOwnerID(request.FileID)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
// note: instead of the final url, give a temp url for upload purpose.
uploadUrl := fmt.Sprintf("%s_temp_upload", filedata.PreviewUrl(request.FileID, fileOwnerID, request.Type))
bucketID := c.S3Config.GetBucketID(request.Type)
enteUrl, err := c.getUploadURL(bucketID, uploadUrl)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &enteUrl.URL, nil
}

40
server/pkg/controller/filedata/s3.go
View File

@@ -7,11 +7,51 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/ente-io/museum/ente"
fileData "github.com/ente-io/museum/ente/filedata"
"github.com/ente-io/stacktrace"
log "github.com/sirupsen/logrus"
stime "time"
)
const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour
func (c *Controller) getUploadURL(dc string, objectKey string) (*ente.UploadURL, error) {
s3Client := c.S3Config.GetS3Client(dc)
r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
Bucket: c.S3Config.GetBucket(dc),
Key: &objectKey,
})
url, err := r.Presign(PreSignedRequestValidityDuration)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
err = c.ObjectCleanupController.AddTempObjectKey(objectKey, dc)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &ente.UploadURL{
ObjectKey: objectKey,
URL: url,
}, nil
}
func (c *Controller) signedUrlGet(dc string, objectKey string) (*ente.UploadURL, error) {
s3Client := c.S3Config.GetS3Client(dc)
r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{
Bucket: c.S3Config.GetBucket(dc),
Key: &objectKey,
})
url, err := r.Presign(PreSignedRequestValidityDuration)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &ente.UploadURL{ObjectKey: objectKey, URL: url}, nil
}
func (c *Controller) downloadObject(ctx context.Context, objectKey string, dc string) (fileData.S3FileMetadata, error) {
var obj fileData.S3FileMetadata
buff := &aws.WriteAtBuffer{}