TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] Switch to new dup implementation (same as mobile) (#4512)

Browse Source
This commit is contained in:
Manav Rathi
2024-12-27 11:17:25 +05:30
committed by GitHub
parent b026020485 6d2a223acf
commit dd7cea1f96
14 changed files with 79 additions and 990 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
web/apps/photos/src/components/PhotoFrame.tsx
View File

@@ -8,7 +8,6 @@ import { EnteFile } from "@/media/file";
import { FileType } from "@/media/file-type";
import type { GalleryBarMode } from "@/new/photos/components/gallery/reducer";
import { TRASH_SECTION } from "@/new/photos/services/collection";
import { PHOTOS_PAGES } from "@ente/shared/constants/pages";
import useMemoSingleThreaded from "@ente/shared/hooks/useMemoSingleThreaded";
import { styled } from "@mui/material";
import PhotoViewer, { type PhotoViewerProps } from "components/PhotoViewer";
@@ -17,14 +16,12 @@ import { GalleryContext } from "pages/gallery";
import PhotoSwipe from "photoswipe";
import { useContext, useEffect, useState } from "react";
import AutoSizer from "react-virtualized-auto-sizer";
import { Duplicate } from "services/deduplicationService";
import {
SelectedState,
SetFilesDownloadProgressAttributesCreator,
} from "types/gallery";
import { handleSelectCreator } from "utils/photoFrame";
import { PhotoList } from "./PhotoList";
import { DedupePhotoList } from "./PhotoList/dedupe";
import PreviewCard from "./pages/gallery/PreviewCard";
const Container = styled("div")`
@@ -60,10 +57,6 @@ export type DisplayFile = EnteFile & {
};
export interface PhotoFrameProps {
page:
| PHOTOS_PAGES.GALLERY
| PHOTOS_PAGES.DEDUPLICATE
| PHOTOS_PAGES.SHARED_ALBUMS;
mode?: GalleryBarMode;
/**
* This is an experimental prop, to see if we can merge the separate
@@ -72,7 +65,6 @@ export interface PhotoFrameProps {
*/
modePlus?: GalleryBarMode | "search";
files: EnteFile[];
duplicates?: Duplicate[];
syncWithRemote: () => Promise<void>;
favItemIds?: Set<number>;
setSelected: (
@@ -96,8 +88,6 @@ export interface PhotoFrameProps {
}
const PhotoFrame = ({
page,
duplicates,
mode,
modePlus,
files,
@@ -476,30 +466,19 @@ const PhotoFrame = ({
return (
<Container>
<AutoSizer>
{({ height, width }) =>
page === PHOTOS_PAGES.DEDUPLICATE ? (
<DedupePhotoList
width={width}
height={height}
getThumbnail={getThumbnail}
duplicates={duplicates}
activeCollectionID={activeCollectionID}
showAppDownloadBanner={showAppDownloadBanner}
/>
) : (
<PhotoList
width={width}
height={height}
getThumbnail={getThumbnail}
mode={mode}
modePlus={modePlus}
displayFiles={displayFiles}
activeCollectionID={activeCollectionID}
activePersonID={activePersonID}
showAppDownloadBanner={showAppDownloadBanner}
/>
)
}
{({ height, width }) => (
<PhotoList
width={width}
height={height}
getThumbnail={getThumbnail}
mode={mode}
modePlus={modePlus}
displayFiles={displayFiles}
activeCollectionID={activeCollectionID}
activePersonID={activePersonID}
showAppDownloadBanner={showAppDownloadBanner}
/>
)}
</AutoSizer>
<PhotoViewer
isOpen={open}

373
web/apps/photos/src/components/PhotoList/dedupe.tsx
View File

@@ -1,373 +0,0 @@
import { EnteFile } from "@/media/file";
import {
GAP_BTW_TILES,
IMAGE_CONTAINER_MAX_HEIGHT,
IMAGE_CONTAINER_MAX_WIDTH,
MIN_COLUMNS,
} from "@/new/photos/components/PhotoList";
import { formattedByteSize } from "@/new/photos/utils/units";
import { FlexWrapper } from "@ente/shared/components/Container";
import { Box, styled } from "@mui/material";
import { t } from "i18next";
import memoize from "memoize-one";
import React, { useEffect, useMemo, useRef, useState } from "react";
import {
VariableSizeList as List,
ListChildComponentProps,
areEqual,
} from "react-window";
import { Duplicate } from "services/deduplicationService";
import {
DATE_CONTAINER_HEIGHT,
SIZE_AND_COUNT_CONTAINER_HEIGHT,
SPACE_BTW_DATES,
} from "components/PhotoList";
export enum ITEM_TYPE {
TIME = "TIME",
FILE = "FILE",
SIZE_AND_COUNT = "SIZE_AND_COUNT",
HEADER = "HEADER",
FOOTER = "FOOTER",
MARKETING_FOOTER = "MARKETING_FOOTER",
OTHER = "OTHER",
}
export interface TimeStampListItem {
itemType: ITEM_TYPE;
items?: EnteFile[];
itemStartIndex?: number;
date?: string;
dates?: {
date: string;
span: number;
}[];
groups?: number[];
item?: any;
id?: string;
height?: number;
fileSize?: number;
fileCount?: number;
}
const ListItem = styled("div")`
display: flex;
justify-content: center;
`;
const getTemplateColumns = (
columns: number,
shrinkRatio: number,
groups?: number[],
): string => {
if (groups) {
// need to confirm why this was there
// const sum = groups.reduce((acc, item) => acc + item, 0);
// if (sum < columns) {
// groups[groups.length - 1] += columns - sum;
// }
return groups
.map(
(x) =>
`repeat(${x}, ${IMAGE_CONTAINER_MAX_WIDTH * shrinkRatio}px)`,
)
.join(` ${SPACE_BTW_DATES}px `);
} else {
return `repeat(${columns},${
IMAGE_CONTAINER_MAX_WIDTH * shrinkRatio
}px)`;
}
};
function getFractionFittableColumns(width: number): number {
return (
(width - 2 * getGapFromScreenEdge(width) + GAP_BTW_TILES) /
(IMAGE_CONTAINER_MAX_WIDTH + GAP_BTW_TILES)
);
}
function getGapFromScreenEdge(width: number) {
if (width > MIN_COLUMNS * IMAGE_CONTAINER_MAX_WIDTH) {
return 24;
} else {
return 4;
}
}
function getShrinkRatio(width: number, columns: number) {
return (
(width -
2 * getGapFromScreenEdge(width) -
(columns - 1) * GAP_BTW_TILES) /
(columns * IMAGE_CONTAINER_MAX_WIDTH)
);
}
const ListContainer = styled(Box)<{
columns: number;
shrinkRatio: number;
groups?: number[];
}>`
display: grid;
grid-template-columns: ${({ columns, shrinkRatio, groups }) =>
getTemplateColumns(columns, shrinkRatio, groups)};
grid-column-gap: ${GAP_BTW_TILES}px;
width: 100%;
color: #fff;
padding: 0 24px;
@media (max-width: ${IMAGE_CONTAINER_MAX_WIDTH * MIN_COLUMNS}px) {
padding: 0 4px;
}
`;
const ListItemContainer = styled(FlexWrapper)<{ span: number }>`
grid-column: span ${(props) => props.span};
`;
const DateContainer = styled(ListItemContainer)`
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
height: ${DATE_CONTAINER_HEIGHT}px;
color: ${({ theme }) => theme.colors.text.muted};
`;
const SizeAndCountContainer = styled(DateContainer)`
margin-top: 1rem;
height: ${SIZE_AND_COUNT_CONTAINER_HEIGHT}px;
`;
interface Props {
height: number;
width: number;
duplicates: Duplicate[];
showAppDownloadBanner: boolean;
getThumbnail: (
file: EnteFile,
index: number,
isScrolling?: boolean,
) => React.JSX.Element;
activeCollectionID: number;
}
interface ItemData {
timeStampList: TimeStampListItem[];
columns: number;
shrinkRatio: number;
renderListItem: (
timeStampListItem: TimeStampListItem,
isScrolling?: boolean,
) => React.JSX.Element;
}
const createItemData = memoize(
(
timeStampList: TimeStampListItem[],
columns: number,
shrinkRatio: number,
renderListItem: (
timeStampListItem: TimeStampListItem,
isScrolling?: boolean,
) => React.JSX.Element,
): ItemData => ({
timeStampList,
columns,
shrinkRatio,
renderListItem,
}),
);
const PhotoListRow = React.memo(
({
index,
style,
isScrolling,
data,
}: ListChildComponentProps<ItemData>) => {
const { timeStampList, columns, shrinkRatio, renderListItem } = data;
return (
<ListItem style={style}>
<ListContainer
columns={columns}
shrinkRatio={shrinkRatio}
groups={timeStampList[index].groups}
>
{renderListItem(timeStampList[index], isScrolling)}
</ListContainer>
</ListItem>
);
},
areEqual,
);
const getTimeStampListFromDuplicates = (duplicates: Duplicate[], columns) => {
const timeStampList: TimeStampListItem[] = [];
for (let index = 0; index < duplicates.length; index++) {
const dupes = duplicates[index];
timeStampList.push({
itemType: ITEM_TYPE.SIZE_AND_COUNT,
fileSize: dupes.size,
fileCount: dupes.files.length,
});
let lastIndex = 0;
while (lastIndex < dupes.files.length) {
timeStampList.push({
itemType: ITEM_TYPE.FILE,
items: dupes.files.slice(lastIndex, lastIndex + columns),
itemStartIndex: index,
});
lastIndex += columns;
}
}
return timeStampList;
};
export function DedupePhotoList({
height,
width,
duplicates,
getThumbnail,
activeCollectionID,
}: Props) {
const [timeStampList, setTimeStampList] = useState<TimeStampListItem[]>([]);
const refreshInProgress = useRef(false);
const shouldRefresh = useRef(false);
const listRef = useRef(null);
const columns = useMemo(() => {
const fittableColumns = getFractionFittableColumns(width);
let columns = Math.floor(fittableColumns);
if (columns < MIN_COLUMNS) {
columns = MIN_COLUMNS;
}
return columns;
}, [width]);
const shrinkRatio = getShrinkRatio(width, columns);
const listItemHeight =
IMAGE_CONTAINER_MAX_HEIGHT * shrinkRatio + GAP_BTW_TILES;
const refreshList = () => {
listRef.current?.resetAfterIndex(0);
};
useEffect(() => {
const main = () => {
if (refreshInProgress.current) {
shouldRefresh.current = true;
return;
}
refreshInProgress.current = true;
const timeStampList = getTimeStampListFromDuplicates(
duplicates,
columns,
);
setTimeStampList(timeStampList);
refreshInProgress.current = false;
if (shouldRefresh.current) {
shouldRefresh.current = false;
setTimeout(main, 0);
}
};
main();
}, [columns, duplicates]);
useEffect(() => {
refreshList();
}, [timeStampList]);
const getItemSize = (timeStampList) => (index) => {
switch (timeStampList[index].itemType) {
case ITEM_TYPE.TIME:
return DATE_CONTAINER_HEIGHT;
case ITEM_TYPE.SIZE_AND_COUNT:
return SIZE_AND_COUNT_CONTAINER_HEIGHT;
case ITEM_TYPE.FILE:
return listItemHeight;
default:
return timeStampList[index].height;
}
};
const generateKey = (index) => {
switch (timeStampList[index].itemType) {
case ITEM_TYPE.FILE:
return `${timeStampList[index].items[0].id}-${
timeStampList[index].items.slice(-1)[0].id
}`;
default:
return `${timeStampList[index].id}-${index}`;
}
};
const renderListItem = (
listItem: TimeStampListItem,
isScrolling: boolean,
) => {
switch (listItem.itemType) {
case ITEM_TYPE.SIZE_AND_COUNT:
return (
/*TODO: Translate the full phrase instead of piecing
together parts like this See:
https://crowdin.com/editor/ente-photos-web/9/enus-de?view=comfortable&filter=basic&value=0#8104
*/
<SizeAndCountContainer span={columns}>
{listItem.fileCount} {t("FILES")},{" "}
{formattedByteSize(listItem.fileSize || 0)} {t("EACH")}
</SizeAndCountContainer>
);
case ITEM_TYPE.FILE: {
const ret = listItem.items.map((item, idx) =>
getThumbnail(
item,
listItem.itemStartIndex + idx,
isScrolling,
),
);
if (listItem.groups) {
let sum = 0;
for (let i = 0; i < listItem.groups.length - 1; i++) {
sum = sum + listItem.groups[i];
ret.splice(
sum,
0,
<div key={`${listItem.items[0].id}-gap-${i}`} />,
);
sum += 1;
}
}
return ret;
}
default:
return listItem.item;
}
};
if (!timeStampList?.length) {
return <></>;
}
const itemData = createItemData(
timeStampList,
columns,
shrinkRatio,
renderListItem,
);
return (
<List
key={`${activeCollectionID}`}
itemData={itemData}
ref={listRef}
itemSize={getItemSize(timeStampList)}
height={height}
width={width}
itemCount={timeStampList.length}
itemKey={generateKey}
overscanCount={3}
useIsScrolling
>
{PhotoListRow}
</List>
);
}

17
web/apps/photos/src/components/PhotoList/index.tsx
View File

@@ -1,5 +1,4 @@
import { EnteFile } from "@/media/file";
import { formattedByteSize } from "@/new/photos/utils/units";
import { FlexWrapper } from "@ente/shared/components/Container";
import { formatDate } from "@ente/shared/time/format";
import { Box, Checkbox, Link, Typography, styled } from "@mui/material";
@@ -26,7 +25,6 @@ import {
import type { PhotoFrameProps } from "components/PhotoFrame";
export const DATE_CONTAINER_HEIGHT = 48;
export const SIZE_AND_COUNT_CONTAINER_HEIGHT = 72;
export const SPACE_BTW_DATES = 44;
const SPACE_BTW_DATES_TO_IMAGE_CONTAINER_WIDTH_RATIO = 0.244;
@@ -38,7 +36,6 @@ const ALBUM_FOOTER_HEIGHT_WITH_REFERRAL = 113;
export enum ITEM_TYPE {
TIME = "TIME",
FILE = "FILE",
SIZE_AND_COUNT = "SIZE_AND_COUNT",
HEADER = "HEADER",
FOOTER = "FOOTER",
MARKETING_FOOTER = "MARKETING_FOOTER",
@@ -143,11 +140,6 @@ const DateContainer = styled(ListItemContainer)`
color: ${({ theme }) => theme.colors.text.muted};
`;
const SizeAndCountContainer = styled(DateContainer)`
margin-top: 1rem;
height: ${SIZE_AND_COUNT_CONTAINER_HEIGHT}px;
`;
const FooterContainer = styled(ListItemContainer)`
margin-bottom: 0.75rem;
@media (max-width: 540px) {
@@ -712,8 +704,6 @@ export function PhotoList({
switch (timeStampList[index].itemType) {
case ITEM_TYPE.TIME:
return DATE_CONTAINER_HEIGHT;
case ITEM_TYPE.SIZE_AND_COUNT:
return SIZE_AND_COUNT_CONTAINER_HEIGHT;
case ITEM_TYPE.FILE:
return listItemHeight;
default:
@@ -842,13 +832,6 @@ export function PhotoList({
{listItem.date}
</DateContainer>
);
case ITEM_TYPE.SIZE_AND_COUNT:
return (
<SizeAndCountContainer span={columns}>
{listItem.fileCount} {t("FILES")},{" "}
{formattedByteSize(listItem.fileSize || 0)} {t("EACH")}
</SizeAndCountContainer>
);
case ITEM_TYPE.FILE: {
const ret = listItem.items.map((item, idx) =>
getThumbnail(

4
web/apps/photos/src/components/Sidebar/index.tsx
View File

@@ -514,7 +514,7 @@ const UtilitySection: React.FC<UtilitySectionProps> = ({ closeSidebar }) => {
await openAccountsManagePasskeysPage();
};
const redirectToDeduplicatePage = () => router.push(PAGES.DEDUPLICATE);
const handleDeduplicate = () => router.push("/duplicates");
const toggleTheme = () =>
setThemeColor(
@@ -573,7 +573,7 @@ const UtilitySection: React.FC<UtilitySectionProps> = ({ closeSidebar }) => {
/>
<EnteMenuItem
variant="secondary"
onClick={redirectToDeduplicatePage}
onClick={handleDeduplicate}
label={t("DEDUPLICATE_FILES")}
/>
<EnteMenuItem

58
web/apps/photos/src/components/pages/dedupe/SelectedFileOptions.tsx
View File

@@ -1,58 +0,0 @@
import { SelectionBar } from "@/base/components/Navbar";
import { AppContext } from "@/new/photos/types/context";
import { FluidContainer } from "@ente/shared/components/Container";
import BackButton from "@mui/icons-material/ArrowBackOutlined";
import CloseIcon from "@mui/icons-material/Close";
import DeleteIcon from "@mui/icons-material/Delete";
import { Box, IconButton, Tooltip } from "@mui/material";
import { t } from "i18next";
import { useContext } from "react";
interface IProps {
deleteFileHelper: () => void;
close: () => void;
count: number;
clearSelection: () => void;
}
export default function DeduplicateOptions({
deleteFileHelper,
close,
count,
clearSelection,
}: IProps) {
const { showMiniDialog } = useContext(AppContext);
const trashHandler = () =>
showMiniDialog({
title: t("trash_files_title"),
message: t("trash_files_message"),
continue: {
text: t("move_to_trash"),
color: "critical",
action: deleteFileHelper,
},
});
return (
<SelectionBar>
<FluidContainer>
{count ? (
<IconButton onClick={clearSelection}>
<CloseIcon />
</IconButton>
) : (
<IconButton onClick={close}>
<BackButton />
</IconButton>
)}
<Box ml={1.5}>{t("selected_count", { selected: count })}</Box>
</FluidContainer>
<Tooltip title={t("delete")}>
<IconButton onClick={trashHandler}>
<DeleteIcon />
</IconButton>
</Tooltip>
</SelectionBar>
);
}

35
web/apps/photos/src/components/pages/gallery/PreviewCard.tsx
View File

@@ -15,10 +15,9 @@ import useLongPress from "@ente/shared/hooks/useLongPress";
import AlbumOutlined from "@mui/icons-material/AlbumOutlined";
import Favorite from "@mui/icons-material/FavoriteRounded";
import PlayCircleOutlineOutlinedIcon from "@mui/icons-material/PlayCircleOutlineOutlined";
import { Tooltip, styled } from "@mui/material";
import { styled } from "@mui/material";
import type { DisplayFile } from "components/PhotoFrame";
import i18n from "i18next";
import { DeduplicateContext } from "pages/deduplicate";
import { GalleryContext } from "pages/gallery";
import React, { useContext, useEffect, useRef, useState } from "react";
import { shouldShowAvatar } from "utils/file";
@@ -235,7 +234,6 @@ const Cont = styled("div")<{ disabled: boolean }>`
export default function PreviewCard(props: IProps) {
const galleryContext = useContext(GalleryContext);
const deduplicateContext = useContext(DeduplicateContext);
const longPressCallback = () => {
onSelect(!selected);
@@ -318,7 +316,7 @@ export default function PreviewCard(props: IProps) {
}
};
const renderFn = () => (
return (
<Cont
key={`thumb-${file.id}}`}
onClick={handleClick}
@@ -372,16 +370,6 @@ export default function PreviewCard(props: IProps) {
<InSelectRangeOverLay
$active={isRangeSelectActive && isInsSelectRange}
/>
{deduplicateContext.isOnDeduplicatePage && (
<FileAndCollectionNameOverlay>
<p>{file.metadata.title}</p>
<p>
{deduplicateContext.collectionNameMap.get(
file.collectionID,
)}
</p>
</FileAndCollectionNameOverlay>
)}
{props?.activeCollectionID === TRASH_SECTION && file.isTrashed && (
<FileAndCollectionNameOverlay>
<p>{formatDateRelative(file.deleteBy / 1000)}</p>
@@ -389,25 +377,6 @@ export default function PreviewCard(props: IProps) {
)}
</Cont>
);
if (deduplicateContext.isOnDeduplicatePage) {
return (
<Tooltip
placement="bottom-start"
enterDelay={300}
enterNextDelay={100}
title={`${
file.metadata.title
} - ${deduplicateContext.collectionNameMap.get(
file.collectionID,
)}`}
>
{renderFn()}
</Tooltip>
);
} else {
return renderFn();
}
}
function formatDateRelative(date: number) {

230
web/apps/photos/src/pages/deduplicate.tsx
View File

@@ -1,230 +0,0 @@
import { stashRedirect } from "@/accounts/services/redirect";
import { ActivityIndicator } from "@/base/components/mui/ActivityIndicator";
import { errorDialogAttributes } from "@/base/components/utils/dialog";
import log from "@/base/log";
import { ALL_SECTION, moveToTrash } from "@/new/photos/services/collection";
import {
getAllLatestCollections,
getLocalCollections,
syncTrash,
} from "@/new/photos/services/collections";
import {
createFileCollectionIDs,
getLocalFiles,
syncFiles,
} from "@/new/photos/services/files";
import { useAppContext } from "@/new/photos/types/context";
import { VerticallyCentered } from "@ente/shared/components/Container";
import { PHOTOS_PAGES as PAGES } from "@ente/shared/constants/pages";
import { ApiError } from "@ente/shared/error";
import useMemoSingleThreaded from "@ente/shared/hooks/useMemoSingleThreaded";
import { SESSION_KEYS, getKey } from "@ente/shared/storage/sessionStorage";
import { styled } from "@mui/material";
import Typography from "@mui/material/Typography";
import { HttpStatusCode } from "axios";
import DeduplicateOptions from "components/pages/dedupe/SelectedFileOptions";
import PhotoFrame from "components/PhotoFrame";
import { t } from "i18next";
import { default as Router, default as router } from "next/router";
import { createContext, useEffect, useState } from "react";
import { Duplicate, getDuplicates } from "services/deduplicationService";
import { SelectedState } from "types/gallery";
import { getSelectedFiles } from "utils/file";
export interface DeduplicateContextType {
isOnDeduplicatePage: boolean;
collectionNameMap: Map<number, string>;
}
export const DeduplicateContext = createContext<DeduplicateContextType>({
isOnDeduplicatePage: false,
collectionNameMap: new Map<number, string>(),
});
export const Info = styled("div")`
padding: 24px;
font-size: 18px;
`;
export default function Deduplicate() {
const { showNavBar, showLoadingBar, hideLoadingBar, showMiniDialog } =
useAppContext();
const [duplicates, setDuplicates] = useState<Duplicate[]>(null);
const [collectionNameMap, setCollectionNameMap] = useState(
new Map<number, string>(),
);
const [selected, setSelected] = useState<SelectedState>({
count: 0,
collectionID: 0,
ownCount: 0,
context: undefined,
});
const closeDeduplication = function () {
Router.push(PAGES.GALLERY);
};
useEffect(() => {
const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
if (!key) {
stashRedirect(PAGES.DEDUPLICATE);
router.push("/");
return;
}
showNavBar(true);
}, []);
useEffect(() => {
syncWithRemote();
}, []);
const syncWithRemote = async () => {
showLoadingBar();
try {
const collections = await getLocalCollections();
const collectionNameMap = new Map<number, string>();
for (const collection of collections) {
collectionNameMap.set(collection.id, collection.name);
}
setCollectionNameMap(collectionNameMap);
const files = await getLocalFiles();
const duplicateFiles = await getDuplicates(
files,
collectionNameMap,
);
const currFileSizeMap = new Map<number, number>();
let toSelectFileIDs: number[] = [];
let count = 0;
for (const dupe of duplicateFiles) {
// select all except first file
toSelectFileIDs = [
...toSelectFileIDs,
...dupe.files.slice(1).map((f) => f.id),
];
count += dupe.files.length - 1;
for (const file of dupe.files) {
currFileSizeMap.set(file.id, dupe.size);
}
}
setDuplicates(duplicateFiles);
const selectedFiles = {
count: count,
ownCount: count,
collectionID: ALL_SECTION,
context: undefined,
};
for (const fileID of toSelectFileIDs) {
selectedFiles[fileID] = true;
}
setSelected(selectedFiles);
} finally {
hideLoadingBar();
}
};
const duplicateFiles = useMemoSingleThreaded(() => {
return (duplicates ?? []).reduce((acc, dupe) => {
return [...acc, ...dupe.files];
}, []);
}, [duplicates]);
const fileToCollectionsMap = useMemoSingleThreaded(() => {
return createFileCollectionIDs(duplicateFiles ?? []);
}, [duplicateFiles]);
const deleteFileHelper = async () => {
try {
showLoadingBar();
const selectedFiles = getSelectedFiles(selected, duplicateFiles);
await moveToTrash(selectedFiles);
// moveToTrash above does an API request, we still need to update
// our local state.
//
// Enhancement: This can be done in a more granular manner. Also, it
// is better to funnel these syncs instead of adding these here and
// there in an ad-hoc manner. For now, this fixes the issue with the
// UI not updating if the user deletes only some of the duplicates.
const collections = await getAllLatestCollections();
await syncFiles(
"normal",
collections,
() => {},
() => {},
);
await syncTrash(collections, () => {});
await syncWithRemote();
} catch (e) {
log.error("Dedup delete failed", e);
await syncWithRemote();
// See: [Note: Chained MiniDialogs]
setTimeout(() => {
showMiniDialog(
errorDialogAttributes(
e instanceof ApiError &&
e.httpStatusCode == HttpStatusCode.Forbidden
? t("not_file_owner_delete_error")
: t("generic_error"),
),
);
}, 0);
} finally {
hideLoadingBar();
}
};
const clearSelection = function () {
setSelected({
count: 0,
collectionID: 0,
ownCount: 0,
context: undefined,
});
};
if (!duplicates) {
return (
<VerticallyCentered>
<ActivityIndicator />
</VerticallyCentered>
);
}
return (
<DeduplicateContext.Provider
value={{
collectionNameMap,
isOnDeduplicatePage: true,
}}
>
{duplicateFiles.length > 0 && (
<Info>{t("DEDUPLICATE_BASED_ON_SIZE")}</Info>
)}
{duplicateFiles.length === 0 ? (
<VerticallyCentered>
<Typography variant="large" color="text.muted">
{t("NO_DUPLICATES_FOUND")}
</Typography>
</VerticallyCentered>
) : (
<PhotoFrame
page={PAGES.DEDUPLICATE}
files={duplicateFiles}
duplicates={duplicates}
syncWithRemote={syncWithRemote}
setSelected={setSelected}
selected={selected}
activeCollectionID={ALL_SECTION}
fileToCollectionsMap={fileToCollectionsMap}
collectionNameMap={collectionNameMap}
selectable={true}
/>
)}
<DeduplicateOptions
deleteFileHelper={deleteFileHelper}
count={selected.count}
close={closeDeduplication}
clearSelection={clearSelection}
/>
</DeduplicateContext.Provider>
);
}

1
web/apps/photos/src/pages/gallery.tsx
View File

@@ -1029,7 +1029,6 @@ export default function Gallery() {
<PeopleEmptyState />
) : (
<PhotoFrame
page={PAGES.GALLERY}
mode={barMode}
modePlus={isInSearchMode ? "search" : barMode}
files={filteredFiles}

1
web/apps/photos/src/pages/shared-albums.tsx
View File

@@ -497,7 +497,6 @@ export default function PublicCollectionGallery() {
/>
<SharedAlbumNavbar onAddPhotos={onAddPhotos} />
<PhotoFrame
page={PAGES.SHARED_ALBUMS}
files={publicFiles}
syncWithRemote={syncWithRemote}
setSelected={setSelected}

176
web/apps/photos/src/services/deduplicationService.ts
View File

@@ -1,176 +0,0 @@
import log from "@/base/log";
import { apiURL } from "@/base/origins";
import { EnteFile } from "@/media/file";
import { metadataHash, type Metadata } from "@/media/file-metadata";
import HTTPService from "@ente/shared/network/HTTPService";
import { getToken } from "@ente/shared/storage/localStorage/helpers";
interface DuplicatesResponse {
duplicates: {
fileIDs: number[];
size: number;
}[];
}
export interface Duplicate {
files: EnteFile[];
size: number;
}
export async function getDuplicates(
files: EnteFile[],
collectionNameMap: Map<number, string>,
) {
try {
const ascDupes = await fetchDuplicateFileIDs();
const descSortedDupes = ascDupes.sort((firstDupe, secondDupe) => {
return secondDupe.size - firstDupe.size;
});
const fileMap = new Map<number, EnteFile>();
for (const file of files) {
fileMap.set(file.id, file);
}
let result: Duplicate[] = [];
for (const dupe of descSortedDupes) {
let duplicateFiles: EnteFile[] = [];
for (const fileID of dupe.fileIDs) {
if (fileMap.has(fileID)) {
duplicateFiles.push(fileMap.get(fileID));
}
}
duplicateFiles = await sortDuplicateFiles(
duplicateFiles,
collectionNameMap,
);
if (duplicateFiles.length > 1) {
result = [
...result,
...getDupesGroupedBySameFileHashes({
files: duplicateFiles,
size: dupe.size,
}),
];
}
}
return result;
} catch (e) {
log.error("failed to get duplicate files", e);
}
}
const hasFileHash = (file: Metadata) => !!metadataHash(file);
function getDupesGroupedBySameFileHashes(dupe: Duplicate) {
const result: Duplicate[] = [];
const fileWithHashes: EnteFile[] = [];
const fileWithoutHashes: EnteFile[] = [];
for (const file of dupe.files) {
if (hasFileHash(file.metadata)) {
fileWithHashes.push(file);
} else {
fileWithoutHashes.push(file);
}
}
if (fileWithHashes.length > 1) {
result.push(
...groupDupesByFileHashes({
files: fileWithHashes,
size: dupe.size,
}),
);
}
if (fileWithoutHashes.length > 1) {
result.push({
files: fileWithoutHashes,
size: dupe.size,
});
}
return result;
}
function groupDupesByFileHashes(dupe: Duplicate) {
const result: Duplicate[] = [];
const filesSortedByFileHash = dupe.files
.map((file) => {
return {
file,
hash: metadataHash(file.metadata),
};
})
.sort((firstFile, secondFile) => {
return firstFile.hash.localeCompare(secondFile.hash);
});
let sameHashFiles: EnteFile[] = [];
sameHashFiles.push(filesSortedByFileHash[0].file);
for (let i = 1; i < filesSortedByFileHash.length; i++) {
if (
areFileHashesSame(
filesSortedByFileHash[i - 1].file.metadata,
filesSortedByFileHash[i].file.metadata,
)
) {
sameHashFiles.push(filesSortedByFileHash[i].file);
} else {
if (sameHashFiles.length > 1) {
result.push({
files: [...sameHashFiles],
size: dupe.size,
});
}
sameHashFiles = [filesSortedByFileHash[i].file];
}
}
if (sameHashFiles.length > 1) {
result.push({
files: sameHashFiles,
size: dupe.size,
});
}
return result;
}
async function fetchDuplicateFileIDs() {
try {
const response = await HTTPService.get(
await apiURL("/files/duplicates"),
null,
{
"X-Auth-Token": getToken(),
},
);
return (response.data as DuplicatesResponse).duplicates;
} catch (e) {
log.error("failed to fetch duplicate file IDs", e);
}
}
async function sortDuplicateFiles(
files: EnteFile[],
collectionNameMap: Map<number, string>,
) {
return files.sort((firstFile, secondFile) => {
const firstCollectionName = collectionNameMap
.get(firstFile.collectionID)
.toLocaleLowerCase();
const secondCollectionName = collectionNameMap
.get(secondFile.collectionID)
.toLocaleLowerCase();
return firstCollectionName.localeCompare(secondCollectionName);
});
}
function areFileHashesSame(firstFile: Metadata, secondFile: Metadata) {
return metadataHash(firstFile) === metadataHash(secondFile);
}

11
web/packages/base/locales/en-US/translation.json
View File

@@ -349,7 +349,6 @@
"leave_shared_album_title": "Leave shared album?",
"leave_shared_album_message": "You will leave the album, and it will stop being visible to you.",
"leave_shared_album": "Yes, leave",
"not_file_owner_delete_error": "You cannot delete files in a shared album",
"confirm_remove_message": "Selected items will be removed from this album. Items which are only in this album will be moved to Uncategorized.",
"confirm_remove_incl_others_message": "Some of the items you are removing were added by other people, and you will lose access to them.",
"oldest": "Oldest",
@@ -418,10 +417,14 @@
"folder": "Folder",
"google_takeout": "Google takeout",
"DEDUPLICATE_FILES": "Deduplicate files",
"NO_DUPLICATES_FOUND": "You have no duplicate files that can be cleared",
"remove_duplicates": "Remove duplicates",
"total_size": "Total size",
"count": "Count",
"deselect_all": "Deselect all",
"no_duplicates": "No duplicates",
"duplicate_group_description": "{{count}} items, {{itemSize}} each",
"remove_duplicates_button_count": "Delete {{count, number}} items",
"FILES": "files",
"EACH": "each",
"DEDUPLICATE_BASED_ON_SIZE": "The following files were clubbed based on their sizes, please review and delete items you believe are duplicates",
"stop_uploads_title": "Stop uploads?",
"stop_uploads_message": "Are you sure that you want to stop all the uploads in progress?",
"yes_stop_uploads": "Yes, stop uploads",

50
web/packages/new/photos/pages/duplicates.tsx
View File

@@ -8,7 +8,6 @@ import {
OverflowMenuOption,
} from "@/base/components/OverflowMenu";
import { Ellipsized2LineTypography } from "@/base/components/Typography";
import { pt } from "@/base/i18n";
import log from "@/base/log";
import { formattedByteSize } from "@/new/photos/utils/units";
import ArrowBackIcon from "@mui/icons-material/ArrowBack";
@@ -26,6 +25,7 @@ import {
Tooltip,
Typography,
} from "@mui/material";
import { t } from "i18next";
import { useRouter } from "next/router";
import React, {
memo,
@@ -57,13 +57,14 @@ import {
import { useAppContext } from "../types/context";
const Page: React.FC = () => {
const { onGenericError } = useAppContext();
const { showNavBar, onGenericError } = useAppContext();
const [state, dispatch] = useReducer(dedupReducer, initialDedupState);
useRedirectIfNeedsCredentials("/duplicates");
useEffect(() => {
showNavBar(false);
dispatch({ type: "analyze" });
void deduceDuplicates()
.then((duplicateGroups) =>
@@ -93,13 +94,13 @@ const Page: React.FC = () => {
}, [state.duplicateGroups, onGenericError]);
const contents = (() => {
switch (state.status) {
switch (state.analysisStatus) {
case undefined:
case "analyzing":
case "started":
return <Loading />;
case "analysisFailed":
case "failed":
return <LoadFailed />;
case "analysisCompleted":
case "completed":
if (state.duplicateGroups.length == 0) {
return <NoDuplicatesFound />;
} else {
@@ -117,8 +118,6 @@ const Page: React.FC = () => {
/>
);
}
default:
return <Loading />;
}
})();
@@ -141,8 +140,8 @@ export default Page;
type SortOrder = "prunableCount" | "prunableSize";
interface DedupState {
/** Status of the screen, between initial state => analysis */
status: undefined | "analyzing" | "analysisFailed" | "analysisCompleted";
/** Status of the analysis ("loading") process. */
analysisStatus: undefined | "started" | "failed" | "completed";
/**
* Groups of duplicates.
*
@@ -188,7 +187,7 @@ type DedupAction =
| { type: "dedupeCompleted"; removedGroupIDs: Set<string> };
const initialDedupState: DedupState = {
status: undefined,
analysisStatus: undefined,
duplicateGroups: [],
sortOrder: "prunableSize",
prunableCount: 0,
@@ -202,9 +201,9 @@ const dedupReducer: React.Reducer<DedupState, DedupAction> = (
) => {
switch (action.type) {
case "analyze":
return { ...state, status: "analyzing" };
return { ...state, analysisStatus: "started" };
case "analysisFailed":
return { ...state, status: "analysisFailed" };
return { ...state, analysisStatus: "failed" };
case "analysisCompleted": {
const duplicateGroups = sortedCopyOfDuplicateGroups(
action.duplicateGroups,
@@ -215,7 +214,7 @@ const dedupReducer: React.Reducer<DedupState, DedupAction> = (
deducePrunableCountAndSize(duplicateGroups);
return {
...state,
status: "analysisCompleted",
analysisStatus: "completed",
duplicateGroups,
selected,
prunableCount,
@@ -295,7 +294,8 @@ const dedupReducer: React.Reducer<DedupState, DedupAction> = (
* Return a copy of the given {@link duplicateGroups}, also sorting them as per
* the given {@link sortOrder}.
*
* Helper method for the reducer */
* Helper method for the reducer.
*/
const sortedCopyOfDuplicateGroups = (
duplicateGroups: DuplicateGroup[],
sortOrder: DedupState["sortOrder"],
@@ -306,7 +306,7 @@ const sortedCopyOfDuplicateGroups = (
: b.prunableCount - a.prunableCount,
);
/** Helper method for the reducer */
/** Helper method for the reducer. */
const deducePrunableCountAndSize = (duplicateGroups: DuplicateGroup[]) => {
const prunableCount = duplicateGroups.reduce(
(sum, { prunableCount, isSelected }) =>
@@ -359,7 +359,7 @@ const Navbar: React.FC<NavbarProps> = ({
<ArrowBackIcon />
</IconButton>
</Box>
<Typography variant="large">{pt("Remove duplicates")}</Typography>
<Typography variant="large">{t("remove_duplicates")}</Typography>
<Stack direction="row" sx={{ gap: "4px" }}>
<SortMenu {...{ sortOrder, onChangeSortOrder }} />
<OptionsMenu {...{ onDeselectAll }} />
@@ -377,7 +377,7 @@ const SortMenu: React.FC<SortMenuProps> = ({
<OverflowMenu
ariaID="duplicates-sort"
triggerButtonIcon={
<Tooltip title={pt("Sort")}>
<Tooltip title={t("sort_by")}>
<SortIcon />
</Tooltip>
}
@@ -386,13 +386,13 @@ const SortMenu: React.FC<SortMenuProps> = ({
endIcon={sortOrder == "prunableSize" ? <DoneIcon /> : undefined}
onClick={() => onChangeSortOrder("prunableSize")}
>
{pt("Total size")}
{t("total_size")}
</OverflowMenuOption>
<OverflowMenuOption
endIcon={sortOrder == "prunableCount" ? <DoneIcon /> : undefined}
onClick={() => onChangeSortOrder("prunableCount")}
>
{pt("Count")}
{t("count")}
</OverflowMenuOption>
</OverflowMenu>
);
@@ -405,7 +405,7 @@ const OptionsMenu: React.FC<OptionsMenuProps> = ({ onDeselectAll }) => (
startIcon={<RemoveCircleOutlineIcon />}
onClick={onDeselectAll}
>
{pt("Deselect all")}
{t("deselect_all")}
</OverflowMenuOption>
</OverflowMenu>
);
@@ -425,7 +425,7 @@ const LoadFailed: React.FC = () => (
const NoDuplicatesFound: React.FC = () => (
<CenteredFill>
<Typography color="text.muted" sx={{ textAlign: "center" }}>
{pt("No duplicates")}
{t("no_duplicates")}
</Typography>
</CenteredFill>
);
@@ -574,7 +574,7 @@ const ListItem: React.FC<ListChildComponentProps<DuplicatesListItemData>> =
}}
>
<Typography color={checked ? "text.base" : "text.muted"}>
{pt(`${count} items, ${itemSize} each`)}
{t("duplicate_group_description", { count, itemSize })}
</Typography>
{/* The size of this Checkbox is 42px. */}
<Checkbox {...{ checked, onChange }} />
@@ -661,7 +661,9 @@ const DeduplicateButton: React.FC<DeduplicateButtonProps> = ({
) : (
<>
<Typography>
{pt(`Delete ${prunableCount} items`)}
{t("remove_duplicates_button_count", {
count: prunableCount,
})}
</Typography>
<Typography variant="small" fontWeight={"normal"}>
{formattedByteSize(prunableSize)}

65
web/packages/new/photos/services/dedup.ts
View File

@@ -32,16 +32,17 @@ export interface DuplicateGroup {
/**
* The underlying file to delete.
*
* This is one of the files from amongst {@link collectionFiles},
* arbitrarily picked to stand in for the entire set of files in the UI.
* This is one of the files from amongst collection files that have the
* same ID, arbitrarily picked to stand in for the entire set of files
* in the UI. The set of the collections to which these files belong is
* retained separately in {@link collectionIDs}.
*/
file: EnteFile;
/**
* All the collection files for the underlying file.
*
* This includes {@link file} too.
* IDs of the collection to which {@link file} and its collection file
* siblings belong.
*/
collectionFiles: EnteFile[];
collectionIDs: Set<number>;
/**
* The name of the collection to which {@link file} belongs.
*
@@ -124,9 +125,9 @@ export const deduceDuplicates = async () => {
);
// Group the filtered collection files by their hashes, keeping only one
// entry per file ID. We also retain all the collections files for a
// particular file ID.
const collectionFilesByFileID = new Map<number, EnteFile[]>();
// entry per file ID. Also retain the IDs of all the collections to which a
// particular file (ID) belongs.
const collectionIDsByFileID = new Map<number, Set<number>>();
const filesByHash = new Map<string, EnteFile[]>();
for (const file of filteredCollectionFiles) {
const hash = metadataHash(file.metadata);
@@ -136,16 +137,14 @@ export const deduceDuplicates = async () => {
continue;
}
const collectionFiles = collectionFilesByFileID.get(file.id);
if (!collectionFiles) {
let collectionIDs = collectionIDsByFileID.get(file.id);
if (!collectionIDs) {
collectionIDsByFileID.set(file.id, (collectionIDs = new Set()));
// This is the first collection file we're seeing for a particular
// file ID, so also create an entry in the filesByHash map.
filesByHash.set(hash, [...(filesByHash.get(hash) ?? []), file]);
}
collectionFilesByFileID.set(file.id, [
...(collectionFiles ?? []),
file,
]);
collectionIDs.add(file.collectionID);
}
// Construct the results from groups that have more than one file with the
@@ -183,15 +182,15 @@ export const deduceDuplicates = async () => {
const collectionName = collectionNameByID.get(
file.collectionID,
);
const collectionFiles = collectionFilesByFileID.get(file.id);
const collectionIDs = collectionIDsByFileID.get(file.id);
// Ignore duplicates for which we do not have a collection. This
// shouldn't really happen though, so retain an assert.
if (!collectionName || !collectionFiles) {
if (!collectionName || !collectionIDs) {
assertionFailed();
return undefined;
}
return { file, collectionFiles, collectionName };
return { file, collectionIDs, collectionName };
})
.filter((item) => !!item);
if (items.length < 2) continue;
@@ -214,7 +213,7 @@ export const deduceDuplicates = async () => {
};
/**
* Remove duplicate groups that the user has retained from those that we
* Remove duplicate groups that the user has selected from those that we
* returned in {@link deduceDuplicates}.
*
* @param duplicateGroups A list of duplicate groups. This is the same list as
@@ -240,34 +239,30 @@ export const removeSelectedDuplicateGroups = async (
// 1. For each selected duplicate group, determine the file to retain.
// 2. Add these to the user owned collections the other files exist in.
// 3. Delete the other files.
//
/* collection ID => files */
const filesToAdd = new Map<number, EnteFile[]>();
/* only one entry per fileID */
const filesToTrash: EnteFile[] = [];
for (const duplicateGroup of selectedDuplicateGroups) {
const retainedItem = duplicateGroupItemToRetain(duplicateGroup);
// Find the existing collection IDs to which this item already belongs.
const existingCollectionIDs = new Set(
retainedItem.collectionFiles.map((cf) => cf.collectionID),
);
// For each item, find all the collections to which any of the files
// (except the file we're retaining) belongs.
const collectionIDs = new Set<number>();
let collectionIDs = new Set<number>();
for (const item of duplicateGroup.items) {
// Skip the item we're retaining.
if (item.file.id == retainedItem.file.id) continue;
// Determine the collections to which any of the item's files belong.
for (const { collectionID } of item.collectionFiles) {
if (!existingCollectionIDs.has(collectionID))
collectionIDs.add(collectionID);
}
collectionIDs = collectionIDs.union(item.collectionIDs);
// Move the item's file to trash.
filesToTrash.push(item.file);
}
// Add the file we're retaining to these (uniqued) collections.
// Skip the existing collection IDs to which this item already belongs.
collectionIDs = collectionIDs.difference(retainedItem.collectionIDs);
// Add the file we're retaining to these collections.
for (const collectionID of collectionIDs) {
filesToAdd.set(collectionID, [
...(filesToAdd.get(collectionID) ?? []),
@@ -283,11 +278,8 @@ export const removeSelectedDuplicateGroups = async (
// Process the adds.
const collections = await getLocalCollections("normal");
const collectionsByID = new Map(collections.map((c) => [c.id, c]));
for (const [collectionID, collectionFiles] of filesToAdd.entries()) {
await addToCollection(
collectionsByID.get(collectionID)!,
collectionFiles,
);
for (const [collectionID, files] of filesToAdd.entries()) {
await addToCollection(collectionsByID.get(collectionID)!, files);
tickProgress();
}
@@ -297,6 +289,7 @@ export const removeSelectedDuplicateGroups = async (
tickProgress();
}
// Sync our local state.
await syncFilesAndCollections();
tickProgress();

1
web/packages/shared/constants/pages.tsx
View File

@@ -13,7 +13,6 @@ export enum PHOTOS_PAGES {
VERIFY = "/verify",
ROOT = "/",
SHARED_ALBUMS = "/shared-albums",
DEDUPLICATE = "/deduplicate",
}
export enum AUTH_PAGES {