From 8f752df4755abc668d69df69844aaf5f6f3289d0 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:05:31 -0500 Subject: [PATCH 01/34] feat(backend): add matchingmode field to Spattachmentdataset --- .../backend/attachment_gw/dataset_views.py | 2 ++ .../migrations/0002_add_matchingmode.py | 18 ++++++++++++++++++ specifyweb/backend/attachment_gw/models.py | 7 +++++++ 3 files changed, 27 insertions(+) create mode 100644 specifyweb/backend/attachment_gw/migrations/0002_add_matchingmode.py diff --git a/specifyweb/backend/attachment_gw/dataset_views.py b/specifyweb/backend/attachment_gw/dataset_views.py index e3d77983fed..aa055ec22b8 100644 --- a/specifyweb/backend/attachment_gw/dataset_views.py +++ b/specifyweb/backend/attachment_gw/dataset_views.py @@ -28,6 +28,7 @@ def datasets_view(request): createdbyagent=request.specify_user_agent, modifiedbyagent=request.specify_user_agent, uploaderstatus="main", + matchingmode=data.get('matchingmode', None), # A bit more flexible than workbench. Handles creating datasets with an uploadplan from the start. uploadplan=json.dumps(data['uploadplan']) if 'uploadplan' in data else None ) @@ -49,6 +50,7 @@ def dataset_view(request, ds: Spattachmentdataset): ds.name = attrs.get('name', ds.name) ds.remarks = attrs.get('remarks', ds.remarks) ds.data = attrs.get('rows', ds.data) + ds.matchingmode = attrs.get('matchingmode', ds.matchingmode) ds.uploadplan = json.dumps(attrs['uploadplan'] if 'uploadplan' in attrs else ds.uploadplan) # Never preserve uploaderstatus. Making it required for all requests. old_status = ds.uploaderstatus diff --git a/specifyweb/backend/attachment_gw/migrations/0002_add_matchingmode.py b/specifyweb/backend/attachment_gw/migrations/0002_add_matchingmode.py new file mode 100644 index 00000000000..000cee0c578 --- /dev/null +++ b/specifyweb/backend/attachment_gw/migrations/0002_add_matchingmode.py @@ -0,0 +1,18 @@ +# Generated manually + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('attachment_gw', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='spattachmentdataset', + name='matchingmode', + field=models.CharField(max_length=32, null=True, default=None), + ), + ] diff --git a/specifyweb/backend/attachment_gw/models.py b/specifyweb/backend/attachment_gw/models.py index 39834301034..81aa3fa0b40 100644 --- a/specifyweb/backend/attachment_gw/models.py +++ b/specifyweb/backend/attachment_gw/models.py @@ -8,6 +8,13 @@ class Spattachmentdataset(Dataset): id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') + matchingmode = models.CharField(max_length=32, null=True, default=None) + + object_response_fields = [ + *Dataset.object_response_fields, + 'matchingmode', + ] + class Meta: db_table = 'attachmentdataset' From c01907941f6e31dd4790067927c2d7d7571dc3b3 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:05:38 -0500 Subject: [PATCH 02/34] feat(attachments): add MappingMode types and localization strings --- .../components/AttachmentsBulkImport/types.ts | 17 ++ .../js_src/lib/localization/attachments.ts | 165 ++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/types.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/types.ts index b39a0addb72..00927065535 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/types.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/types.ts @@ -10,6 +10,18 @@ import type { PartialAttachmentUploadSpec } from './Import'; import type { staticAttachmentImportPaths } from './importPaths'; import type { keyLocalizationMapAttachment } from './utils'; +export type MatchingMode = 'filename' | 'mappingFile'; + +export type MappingFileColumns = { + readonly matchValueColumnIndex: number; + readonly fileNameColumnIndex: number; +}; + +export type MappingFileRow = { + readonly matchValue: string; + readonly fileName: string; +}; + export type UploadAttachmentSpec = { readonly token: string; readonly attachmentLocation: string; @@ -62,6 +74,10 @@ export type BoundFile = Pick; export type UnBoundFile = { readonly file: BoundFile | File; readonly parsedName?: string; + /** Match value from mapping file CSV (used in mapping-file mode) */ + readonly mappingMatchValue?: string; + /** File name from mapping file CSV (used in mapping-file mode) */ + readonly mappingFileName?: string; }; export type AttachmentWorkProgress = { @@ -102,6 +118,7 @@ export type AttachmentDatasetBrief = DatasetBriefBase & { | 'uploading' | 'uploadInterrupted' | 'validating'; + readonly matchingmode?: MatchingMode | null; }; export type AttachmentDataSetPlan = AttachmentDatasetBrief & { diff --git a/specifyweb/frontend/js_src/lib/localization/attachments.ts b/specifyweb/frontend/js_src/lib/localization/attachments.ts index d231d8d5069..48bb7fde364 100644 --- a/specifyweb/frontend/js_src/lib/localization/attachments.ts +++ b/specifyweb/frontend/js_src/lib/localization/attachments.ts @@ -920,4 +920,169 @@ export const attachmentsText = createDictionary({ 'Ovo kontrolira hoće li se novi privitci dodani u ovu kolekciju prema zadanim postavkama označavati kao "Javni". Javni privitci automatski će biti vidljivi na Navedite web portalu. Ova se postavka može poništiti za svaki pojedinačni privitak i ne utječe na postojeće privitke.', nb: 'Dette kontrollerer om nye vedlegg som legges til i denne samlingen skal flagges som «Offentlige» som standard. Offentlige vedlegg vil automatisk være synlige på en Specify-nettportal. Denne innstillingen kan overstyres for hvert vedlegg og påvirker ikke eksisterende vedlegg.', }, + matchingMode: { + 'en-us': 'Matching Mode', + 'de-ch': 'Abgleichsmodus', + 'es-es': 'Modo de coincidencia', + 'fr-fr': 'Mode de correspondance', + 'ru-ru': 'Режим сопоставления', + 'uk-ua': 'Режим зіставлення', + 'pt-br': 'Modo de correspondência', + 'hr-hr': 'Način podudaranja', + nb: 'Samsvarsmodus', + }, + matchByFilename: { + 'en-us': 'Match by filename', + 'de-ch': 'Nach Dateiname abgleichen', + 'es-es': 'Coincidir por nombre de archivo', + 'fr-fr': 'Correspondance par nom de fichier', + 'ru-ru': 'Сопоставление по имени файла', + 'uk-ua': 'Зіставлення за назвою файлу', + 'pt-br': 'Corresponder por nome de arquivo', + 'hr-hr': 'Podudaranje prema nazivu datoteke', + nb: 'Samsvar etter filnavn', + }, + matchByMappingFile: { + 'en-us': 'Match by mapping file', + 'de-ch': 'Nach Zuordnungsdatei abgleichen', + 'es-es': 'Coincidir por archivo de mapeo', + 'fr-fr': 'Correspondance par fichier de mappage', + 'ru-ru': 'Сопоставление по файлу маппинга', + 'uk-ua': 'Зіставлення за файлом зіставлення', + 'pt-br': 'Corresponder por arquivo de mapeamento', + 'hr-hr': 'Podudaranje prema datoteci mapiranja', + nb: 'Samsvar etter tilordningsfil', + }, + fileMissing: { + 'en-us': 'File Missing', + 'de-ch': 'Datei fehlt', + 'es-es': 'Archivo faltante', + 'fr-fr': 'Fichier manquant', + 'ru-ru': 'Файл отсутствует', + 'uk-ua': 'Файл відсутній', + 'pt-br': 'Arquivo ausente', + 'hr-hr': 'Datoteka nedostaje', + nb: 'Fil mangler', + }, + notInMappingFile: { + 'en-us': 'Not in Mapping File', + 'de-ch': 'Nicht in der Zuordnungsdatei', + 'es-es': 'No está en el archivo de mapeo', + 'fr-fr': 'Pas dans le fichier de mappage', + 'ru-ru': 'Нет в файле маппинга', + 'uk-ua': 'Немає у файлі зіставлення', + 'pt-br': 'Não está no arquivo de mapeamento', + 'hr-hr': 'Nije u datoteci mapiranja', + nb: 'Ikke i tilordningsfil', + }, + duplicateInMappingFile: { + 'en-us': 'Duplicate in Mapping File', + 'de-ch': 'Duplikat in der Zuordnungsdatei', + 'es-es': 'Duplicado en el archivo de mapeo', + 'fr-fr': 'Doublon dans le fichier de mappage', + 'ru-ru': 'Дубликат в файле маппинга', + 'uk-ua': 'Дублікат у файлі зіставлення', + 'pt-br': 'Duplicado no arquivo de mapeamento', + 'hr-hr': 'Duplikat u datoteci mapiranja', + nb: 'Duplikat i tilordningsfil', + }, + selectMatchValueColumn: { + 'en-us': 'Match Value Column', + 'de-ch': 'Spalte für Abgleichswert', + 'es-es': 'Columna de valor de coincidencia', + 'fr-fr': 'Colonne de valeur de correspondance', + 'ru-ru': 'Колонка значения сопоставления', + 'uk-ua': 'Стовпець значення зіставлення', + 'pt-br': 'Coluna de valor de correspondência', + 'hr-hr': 'Stupac vrijednosti podudaranja', + nb: 'Kolonne for samsvarsverdi', + }, + selectFileNameColumn: { + 'en-us': 'File Name Column', + 'de-ch': 'Spalte für Dateinamen', + 'es-es': 'Columna de nombre de archivo', + 'fr-fr': 'Colonne de nom de fichier', + 'ru-ru': 'Колонка имени файла', + 'uk-ua': 'Стовпець назви файлу', + 'pt-br': 'Coluna de nome do arquivo', + 'hr-hr': 'Stupac naziva datoteke', + nb: 'Kolonne for filnavn', + }, + mappingFileLoaded: { + 'en-us': 'Mapping file loaded ({count:number} rows)', + 'de-ch': 'Zuordnungsdatei geladen ({count:number} Zeilen)', + 'es-es': 'Archivo de mapeo cargado ({count:number} filas)', + 'fr-fr': 'Fichier de mappage chargé ({count:number} lignes)', + 'ru-ru': 'Файл маппинга загружен ({count:number} строк)', + 'uk-ua': 'Файл зіставлення завантажено ({count:number} рядків)', + 'pt-br': 'Arquivo de mapeamento carregado ({count:number} linhas)', + 'hr-hr': 'Datoteka mapiranja učitana ({count:number} redaka)', + nb: 'Tilordningsfil lastet ({count:number} rader)', + }, + mappingFileColumnsInfo: { + 'en-us': 'Match: {matchColumn:string} → File: {fileColumn:string}', + 'de-ch': 'Abgleich: {matchColumn:string} → Datei: {fileColumn:string}', + 'es-es': 'Coincidencia: {matchColumn:string} → Archivo: {fileColumn:string}', + 'fr-fr': 'Correspondance: {matchColumn:string} → Fichier: {fileColumn:string}', + 'ru-ru': 'Сопоставление: {matchColumn:string} → Файл: {fileColumn:string}', + 'uk-ua': 'Зіставлення: {matchColumn:string} → Файл: {fileColumn:string}', + 'pt-br': 'Correspondência: {matchColumn:string} → Arquivo: {fileColumn:string}', + 'hr-hr': 'Podudaranje: {matchColumn:string} → Datoteka: {fileColumn:string}', + nb: 'Samsvar: {matchColumn:string} → Fil: {fileColumn:string}', + }, + chooseMatchingMode: { + 'en-us': 'Choose how attachment files should be matched to database records.', + 'de-ch': 'Wählen Sie, wie Anhangsdateien mit Datenbankeinträgen abgeglichen werden sollen.', + 'es-es': 'Elija cómo se deben emparejar los archivos adjuntos con los registros de la base de datos.', + 'fr-fr': "Choisissez comment les pièces jointes doivent être associées aux enregistrements de la base de données.", + 'ru-ru': 'Выберите способ сопоставления файлов вложений с записями базы данных.', + 'uk-ua': 'Виберіть спосіб зіставлення файлів вкладень із записами бази даних.', + 'pt-br': 'Escolha como os arquivos anexos devem ser combinados com os registros do banco de dados.', + 'hr-hr': 'Odaberite kako se datoteke privitaka trebaju podudarati s zapisima baze podataka.', + nb: 'Velg hvordan vedleggsfiler skal matches mot databaseposter.', + }, + matchByFilenameDescription: { + 'en-us': 'Parse the record identifier from each filename using a field formatter. Works best with structured, numeric fields like Catalog Number.', + 'de-ch': 'Analysieren Sie die Datensatzkennung aus jedem Dateinamen mit einem Feldformatierer. Funktioniert am besten mit strukturierten, numerischen Feldern.', + 'es-es': 'Analice el identificador de registro de cada nombre de archivo utilizando un formateador de campo. Funciona mejor con campos numéricos estructurados.', + 'fr-fr': "Analysez l'identifiant d'enregistrement à partir de chaque nom de fichier à l'aide d'un formateur de champ. Fonctionne mieux avec des champs numériques structurés.", + 'ru-ru': 'Извлеките идентификатор записи из каждого имени файла с помощью форматировщика полей. Лучше всего работает со структурированными числовыми полями.', + 'uk-ua': 'Витягніть ідентифікатор запису з кожної назви файлу за допомогою форматувальника полів. Найкраще працює зі структурованими числовими полями.', + 'pt-br': 'Analise o identificador de registro de cada nome de arquivo usando um formatador de campo. Funciona melhor com campos numéricos estruturados.', + 'hr-hr': 'Analizirajte identifikator zapisa iz svakog naziva datoteke pomoću formatatora polja. Najbolje radi s strukturiranim, numeričkim poljima.', + nb: 'Analyser postidentifikatoren fra hvert filnavn ved hjelp av en feltformaterer. Fungerer best med strukturerte, numeriske felt.', + }, + matchByMappingFileDescription: { + 'en-us': 'Use an explicit CSV mapping file that lists which record each file belongs to. No filename parsing required — works with any field format, including free text.', + 'de-ch': 'Verwenden Sie eine explizite CSV-Zuordnungsdatei, die auflistet, zu welchem Datensatz jede Datei gehört. Keine Dateinamenanalyse erforderlich.', + 'es-es': 'Use un archivo de mapeo CSV explícito que indique a qué registro pertenece cada archivo. No se requiere análisis de nombres de archivo.', + 'fr-fr': "Utilisez un fichier CSV de mappage explicite qui répertorie à quel enregistrement appartient chaque fichier. Aucune analyse du nom de fichier requise.", + 'ru-ru': 'Используйте явный CSV-файл сопоставления, в котором указано, к какой записи относится каждый файл. Анализ имени файла не требуется.', + 'uk-ua': 'Використовуйте явний CSV-файл зіставлення, у якому вказано, до якого запису належить кожен файл. Аналіз назви файлу не потрібен.', + 'pt-br': 'Use um arquivo CSV de mapeamento explícito que lista a qual registro cada arquivo pertence. Nenhuma análise de nome de arquivo é necessária.', + 'hr-hr': 'Koristite eksplicitnu CSV datoteku mapiranja koja navodi kojem zapisu pripada svaka datoteka. Nije potrebna analiza naziva datoteke.', + nb: 'Bruk en eksplisitt CSV-tilordningsfil som viser hvilken post hver fil tilhører. Ingen filnavnanalyse kreves.', + }, + matchValue: { + 'en-us': 'Match Value', + 'de-ch': 'Abgleichswert', + 'es-es': 'Valor de coincidencia', + 'fr-fr': 'Valeur de correspondance', + 'ru-ru': 'Значение сопоставления', + 'uk-ua': 'Значення зіставлення', + 'pt-br': 'Valor de correspondência', + 'hr-hr': 'Vrijednost podudaranja', + nb: 'Samsvarsverdi', + }, + csvFileName: { + 'en-us': 'CSV File Name', + 'de-ch': 'CSV-Dateiname', + 'es-es': 'Nombre de archivo CSV', + 'fr-fr': 'Nom du fichier CSV', + 'ru-ru': 'Имя файла CSV', + 'uk-ua': 'Назва файлу CSV', + 'pt-br': 'Nome do arquivo CSV', + 'hr-hr': 'Naziv CSV datoteke', + nb: 'CSV-filnavn', + }, } as const); From 2f43c12a05fd4690a2613a56901155ab61c0a86e Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:05:46 -0500 Subject: [PATCH 03/34] feat(attachments): add crossReferenceMappingFiles and prepareMappingFileSelection utilities --- .../__tests__/utils.test.ts | 391 ++++++++++++++++++ .../components/AttachmentsBulkImport/utils.ts | 155 ++++++- 2 files changed, 542 insertions(+), 4 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts index 48584ad4fea..1b7aeb3da31 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts @@ -13,9 +13,11 @@ import { import { syncFieldFormat } from '../../Formatters/fieldFormat'; import type { PartialUploadableFileSpec, UnBoundFile } from '../types'; import { + crossReferenceMappingFiles, inferDeletedAttachments, inferUploadedAttachments, matchFileSpec, + prepareMappingFileSelection, resolveFileNames, } from '../utils'; @@ -354,3 +356,392 @@ test('reconstruct deleting attachment spec', () => { ]; expect(inferDeletedAttachments(queryResults, files)).toMatchSnapshot(); }); + +describe('crossReferenceMappingFiles', () => { + const CSV: RA<{ readonly matchValue: string; readonly fileName: string }> = [ + { matchValue: '000271806', fileName: '271806.jpg' }, + { matchValue: '000601146', fileName: '601146.jpg' }, + { matchValue: '000601146', fileName: '601146_2.jpg' }, + { matchValue: '000273074', fileName: '273074.jpg' }, + ]; + + const makePlaceholder = (name: string, mv: string) => + ({ + uploadFile: { + file: { name, size: 0, type: '' }, + parsedName: mv, + mappingMatchValue: mv, + }, + status: { type: 'cancelled', reason: 'fileMissing' }, + }) as PartialUploadableFileSpec; + + const makeReal = (name: string, mv: string) => + ({ + uploadFile: { + file: { name, size: 91, type: 'image/jpeg' }, + parsedName: mv, + mappingMatchValue: mv, + }, + }) as PartialUploadableFileSpec; + + test('initial seeding produces all placeholders', () => { + const result = crossReferenceMappingFiles([], CSV); + expect(result).toHaveLength(4); + expect(result.every((r) => r.status?.reason === 'fileMissing')).toBe(true); + }); + + test('adding one real file replaces corresponding placeholder', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const result = crossReferenceMappingFiles( + [...seeded, makeReal('601146.jpg', '000601146')], + CSV + ); + expect(result).toHaveLength(4); + expect( + result.find((r) => r.uploadFile.file.name === '601146.jpg') + ?.uploadFile.file.size + ).toBe(91); + expect( + result.filter((r) => r.status?.reason === 'fileMissing') + ).toHaveLength(3); + }); + + test('adding all files removes all placeholders', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const result = crossReferenceMappingFiles( + [ + ...seeded, + makeReal('271806.jpg', '000271806'), + makeReal('601146.jpg', '000601146'), + makeReal('601146_2.jpg', '000601146'), + makeReal('273074.jpg', '000273074'), + ], + CSV + ); + expect(result).toHaveLength(4); + expect( + result.every( + (r) => !r.status || r.status.reason !== 'fileMissing' + ) + ).toBe(true); + expect(result.every((r) => r.uploadFile.file.size === 91)).toBe(true); + }); + + test('batch processing produces no duplicates', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const batch1 = crossReferenceMappingFiles( + [ + ...seeded, + makeReal('271806.jpg', '000271806'), + makeReal('601146.jpg', '000601146'), + ], + CSV + ); + expect(batch1).toHaveLength(4); + + const batch2 = crossReferenceMappingFiles( + [ + ...batch1, + makeReal('601146_2.jpg', '000601146'), + makeReal('273074.jpg', '000273074'), + ], + CSV + ); + expect(batch2).toHaveLength(4); + expect(batch2.every((r) => r.uploadFile.file.size === 91)).toBe(true); + }); + + test('file not in CSV is flagged', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const result = crossReferenceMappingFiles( + [...seeded, makeReal('unknown.jpg', '?')], + CSV + ); + expect( + result.find((r) => r.uploadFile.file.name === 'unknown.jpg')?.status + ?.reason + ).toBe('notInMappingFile'); + }); + + test('duplicate filenames in CSV produce exactly one row', () => { + const csv2: RA<{ readonly matchValue: string; readonly fileName: string }> = + [ + { matchValue: 'A', fileName: 'dup.jpg' }, + { matchValue: 'B', fileName: 'dup.jpg' }, + ]; + const result = crossReferenceMappingFiles( + [...crossReferenceMappingFiles([], csv2), makeReal('dup.jpg', 'A')], + csv2 + ); + expect(result).toHaveLength(1); + expect(result[0].uploadFile.file.size).toBe(91); + }); +}); + +describe('prepareMappingFileSelection', () => { + const CSV: RA<{ readonly matchValue: string; readonly fileName: string }> = [ + { matchValue: '000271806', fileName: '271806.jpg' }, + { matchValue: '000601146', fileName: '601146.jpg' }, + { matchValue: '000601146', fileName: '601146_2.jpg' }, + ]; + + const makePlaceholder = (name: string, mv: string) => + ({ + uploadFile: { + file: { name, size: 0, type: '' }, + parsedName: mv, + mappingMatchValue: mv, + }, + status: { type: 'cancelled', reason: 'fileMissing' }, + }) as PartialUploadableFileSpec; + + const makeReal = (name: string, mv: string) => + ({ + uploadFile: { + file: { name, size: 91, type: 'image/jpeg' }, + parsedName: mv, + mappingMatchValue: mv, + }, + }) as PartialUploadableFileSpec; + + test('replaces seeded placeholders with matching uploaded files', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + seeded, + [makeReal('601146.jpg', '000601146')], + CSV + ); + + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(3); + expect( + resolvedFiles.find((r) => r.uploadFile.file.name === '601146.jpg') + ?.uploadFile.file.size + ).toBe(91); + expect( + resolvedFiles.filter((r) => r.status?.reason === 'fileMissing') + ).toHaveLength(2); + }); + + test('keeps uploaded files that are already real while preserving missing placeholders', () => { + const seeded = crossReferenceMappingFiles([], CSV); + const { resolvedFiles } = prepareMappingFileSelection( + [ + ...seeded, + makeReal('271806.jpg', '000271806'), + makePlaceholder('601146_2.jpg', '000601146'), + ], + [makeReal('601146.jpg', '000601146')], + CSV + ); + + expect(resolvedFiles).toHaveLength(3); + expect(resolvedFiles.every((r) => r.uploadFile.file.size === 91 || r.status?.reason === 'fileMissing')).toBe(true); + expect( + resolvedFiles.filter((r) => r.uploadFile.file.name === '601146.jpg') + ).toHaveLength(1); + }); +}); + +describe('end-to-end: seeding then full file selection', () => { + // Simulates the real user flow: CSV uploaded → placeholders seeded → user selects files + const FULL_CSV: RA<{ + readonly matchValue: string; + readonly fileName: string; + }> = [ + { matchValue: '000271806', fileName: '271806.jpg' }, + { matchValue: '000601146', fileName: '601146.jpg' }, + { matchValue: '000601146', fileName: '601146_2.jpg' }, + { matchValue: '000273074', fileName: '273074.jpg' }, + { matchValue: '000687972', fileName: '687972.jpg' }, + { matchValue: '000601108', fileName: '601108.jpg' }, + { matchValue: '000728604', fileName: '728604.jpg' }, + { matchValue: '000466309', fileName: '466309.jpg' }, + { matchValue: '000475938', fileName: '475938.jpg' }, + { matchValue: '000855732', fileName: '855732.jpg' }, + ]; + + const EXTRA_FILES = ['601766.jpg', '601766_2.jpg', '601766_3.jpg']; + + const allFileNames = [...FULL_CSV.map((r) => r.fileName), ...EXTRA_FILES]; + + const makeRealFile = ( + name: string, + mv: string | undefined + ): PartialUploadableFileSpec => + ({ + uploadFile: { + file: { name, size: 91, type: 'image/jpeg' } as File, + parsedName: mv, + mappingMatchValue: mv, + mappingFileName: mv !== undefined ? name : undefined, + }, + }) as PartialUploadableFileSpec; + + const countNames = ( + files: RA + ): Map => { + const m = new Map(); + for (const f of files) + m.set(f.uploadFile.file.name, (m.get(f.uploadFile.file.name) ?? 0) + 1); + return m; + }; + + test('1: seeding then selecting all files — every name once, mappingMatchValue set', () => { + const seeded = crossReferenceMappingFiles([], FULL_CSV); + expect(seeded).toHaveLength(10); + + const filesToResolve = allFileNames.map((name) => { + const csvRow = FULL_CSV.find((r) => r.fileName === name); + return makeRealFile(name, csvRow?.matchValue); + }); + + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + seeded, + filesToResolve, + FULL_CSV + ); + + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(13); + + const counts = countNames(resolvedFiles); + for (const [, c] of counts) expect(c).toBe(1); + + const csvNames = new Set(FULL_CSV.map((r) => r.fileName)); + for (const r of resolvedFiles) { + if (csvNames.has(r.uploadFile.file.name)) { + expect(r.uploadFile.mappingMatchValue).toBeTruthy(); + expect(r.uploadFile.parsedName).toBeTruthy(); + } else { + expect(r.status?.reason).toBe('notInMappingFile'); + } + } + }); + + test('2: selecting all files on empty rows (no prior seeding)', () => { + const filesToResolve = allFileNames.map((name) => { + const csvRow = FULL_CSV.find((r) => r.fileName === name); + return makeRealFile(name, csvRow?.matchValue); + }); + + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + [], + filesToResolve, + FULL_CSV + ); + + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(13); + + const counts = countNames(resolvedFiles); + for (const [, c] of counts) expect(c).toBe(1); + }); + + test('3: reload scenario — saved rows without mappingMatchValue + re-select files', () => { + // Old saved rows: plain objects, File-like, NO mappingMatchValue + const savedRows = FULL_CSV.map( + (row) => + ({ + uploadFile: { + file: { name: row.fileName, size: 91, type: 'image/jpeg' }, + parsedName: row.matchValue, + }, + }) as PartialUploadableFileSpec + ); + + const newFiles = FULL_CSV.map((row) => + makeRealFile(row.fileName, row.matchValue) + ); + + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + savedRows, + newFiles, + FULL_CSV + ); + + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(10); + + for (const r of resolvedFiles) { + expect(r.uploadFile.mappingMatchValue).toBeTruthy(); + } + + const counts = countNames(resolvedFiles); + for (const [, c] of counts) expect(c).toBe(1); + }); + + test('4: crossReferenceMappingFiles with old row (no mappingMatchValue) + new file → mappingMatchValue wins', () => { + const oldRow = { + uploadFile: { + file: { name: '271806.jpg', size: 91, type: 'image/jpeg' }, + parsedName: '000271806', + }, + } as PartialUploadableFileSpec; + + const newFile = makeRealFile('271806.jpg', '000271806'); + + const result = crossReferenceMappingFiles([oldRow, newFile], FULL_CSV); + + const matches = result.filter( + (r) => r.uploadFile.file.name === '271806.jpg' + ); + expect(matches).toHaveLength(1); + expect(matches[0].uploadFile.mappingMatchValue).toBe('000271806'); + }); + + test('5: every output row has mappingMatchValue after cross-reference with mixed data', () => { + const seeded = crossReferenceMappingFiles([], FULL_CSV); + const realFiles = FULL_CSV.map((row) => + makeRealFile(row.fileName, row.matchValue) + ); + + const result = crossReferenceMappingFiles( + [...seeded, ...realFiles], + FULL_CSV + ); + + expect(result).toHaveLength(10); + for (const r of result) { + expect(r.uploadFile.mappingMatchValue).toBeTruthy(); + } + }); +}); + +describe('crossReferenceMappingFiles CSV always wins matchValue', () => { + test('CSV matchValue overrides row with undefined mappingMatchValue', () => { + const row = { + uploadFile: { + file: { name: 'test.jpg', size: 91, type: 'image/jpeg' }, + parsedName: 'oldValue', + }, + } as PartialUploadableFileSpec; + + const csv = [{ fileName: 'test.jpg', matchValue: 'newValue' }]; + const result = crossReferenceMappingFiles([row], csv); + + expect(result).toHaveLength(1); + expect(result[0].uploadFile.mappingMatchValue).toBe('newValue'); + expect(result[0].uploadFile.parsedName).toBe('newValue'); + expect(result[0].uploadFile.mappingFileName).toBe('test.jpg'); + }); + + test('CSV matchValue overrides row with different mappingMatchValue', () => { + const row = { + uploadFile: { + file: { name: 'test.jpg', size: 91, type: 'image/jpeg' }, + parsedName: 'oldValue', + mappingMatchValue: 'oldValue', + mappingFileName: 'old.jpg', + }, + } as PartialUploadableFileSpec; + + const csv = [{ fileName: 'test.jpg', matchValue: 'CORRECT_VALUE' }]; + const result = crossReferenceMappingFiles([row], csv); + + expect(result).toHaveLength(1); + expect(result[0].uploadFile.mappingMatchValue).toBe('CORRECT_VALUE'); + expect(result[0].uploadFile.parsedName).toBe('CORRECT_VALUE'); + expect(result[0].uploadFile.mappingFileName).toBe('test.jpg'); + }); +}); diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index 7030ecf7e33..f7cc3a51885 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -35,7 +35,11 @@ import { makeQueryField } from '../QueryBuilder/fromTree'; import type { QueryFieldWithPath } from '../Statistics/types'; import type { AttachmentUploadSpec } from './Import'; import { staticAttachmentImportPaths } from './importPaths'; -import type { AttachmentStatus, PartialUploadableFileSpec } from './types'; +import type { + AttachmentStatus, + MappingFileRow, + PartialUploadableFileSpec, +} from './types'; export type ResolvedAttachmentRecord = | State< @@ -63,10 +67,16 @@ const resolveAttachmentMatch = ( export function resolveAttachmentRecord( matchedId: RA | undefined, disambiguated: number | undefined, - parsedName: string | undefined + parsedName: string | undefined, + isMappingMode: boolean = false ): ResolvedAttachmentRecord { - if (parsedName === undefined) - return { type: 'invalid', reason: 'incorrectFormatter' }; + if (parsedName === undefined) { + // In mapping mode, undefined parsedName means not in mapping file + return { + type: 'invalid', + reason: isMappingMode ? 'notInMappingFile' : 'incorrectFormatter', + }; + } if (matchedId === undefined) return { type: 'valid', reason: 'correctlyFormatted' }; return resolveAttachmentMatch(matchedId, disambiguated); @@ -157,6 +167,13 @@ type MatchSelectedFiles = { readonly resolvedFiles: RA; readonly duplicateFiles: RA; }; + +export const isMappingFilePlaceholder = ( + uploadable: PartialUploadableFileSpec +): boolean => + uploadable.status?.type === 'cancelled' && + uploadable.status.reason === 'fileMissing'; + export const matchSelectedFiles = ( previousUploadables: RA, filesToResolve: RA @@ -222,6 +239,133 @@ export const matchSelectedFiles = ( } ); +export const prepareMappingFileSelection = ( + previousUploadables: RA, + filesToResolve: RA, + mappingData: RA +): MatchSelectedFiles => { + const previousRealFiles = previousUploadables.filter( + (uploadable) => !isMappingFilePlaceholder(uploadable) + ); + const { resolvedFiles, duplicateFiles } = matchSelectedFiles( + previousRealFiles, + filesToResolve + ); + + return { + resolvedFiles: crossReferenceMappingFiles(resolvedFiles, mappingData), + duplicateFiles, + }; +}; + +/** + * Cross-reference uploaded files against the mapping file CSV data. + * Builds the result from scratch: for each CSV row, picks the best + * available file (real upload > placeholder). Flags leftover files + * not in the mapping. Guarantees no duplicate filenames. + */ +export function crossReferenceMappingFiles( + uploadableFiles: RA, + mappingData: RA +): RA { + // Build a filename → best-row map (prefer real files over placeholders) + const byName = new Map(); + for (const f of uploadableFiles) { + const existing = byName.get(f.uploadFile.file.name); + if (existing === undefined) { + byName.set(f.uploadFile.file.name, f); + } else { + // Keep the "better" entry: real file > placeholder, + // File instance > plain object (serialized from server), + // row with mappingMatchValue > row without + const existingIsPlaceholder = + existing.status?.type === 'cancelled' && + existing.status.reason === 'fileMissing'; + const newIsPlaceholder = + f.status?.type === 'cancelled' && f.status.reason === 'fileMissing'; + if ( + (existingIsPlaceholder && !newIsPlaceholder) || + (!(existing.uploadFile.file instanceof File) && + f.uploadFile.file instanceof File) + ) { + byName.set(f.uploadFile.file.name, f); + } + } + } + + // Count duplicates in the CSV for flagging + const mappingFileNameCounts = new Map(); + for (const row of mappingData) { + mappingFileNameCounts.set( + row.fileName, + (mappingFileNameCounts.get(row.fileName) ?? 0) + 1 + ); + } + + const result: RA = []; + const matchedFileNames = new Set(); + + // For each CSV row, emit the best available file or a placeholder + for (const row of mappingData) { + const existing = byName.get(row.fileName); + const isDuplicateInCsv = (mappingFileNameCounts.get(row.fileName) ?? 0) > 1; + + if (existing !== undefined) { + // Only emit once per filename even if CSV has duplicate rows + if (matchedFileNames.has(row.fileName)) continue; + matchedFileNames.add(row.fileName); + result.push({ + ...existing, + uploadFile: { + ...existing.uploadFile, + // Always take the match value from the CSV – it is the source of truth + parsedName: row.matchValue, + mappingMatchValue: row.matchValue, + mappingFileName: row.fileName, + }, + ...(isDuplicateInCsv && existing.attachmentId === undefined + ? { + status: { + type: 'cancelled' as const, + reason: 'duplicateInMappingFile' as const, + }, + } + : {}), + }); + } else { + // File not yet uploaded — add placeholder (once per filename) + if (matchedFileNames.has(row.fileName)) continue; + matchedFileNames.add(row.fileName); + result.push({ + uploadFile: { + file: { name: row.fileName, size: 0, type: '' }, + parsedName: row.matchValue, + mappingMatchValue: row.matchValue, + }, + status: { + type: 'cancelled' as const, + reason: 'fileMissing' as const, + }, + }); + } + } + + // Flag any uploaded files not referenced by the CSV + for (const [name, file] of byName) { + if (!matchedFileNames.has(name)) { + result.push({ + ...file, + status: { + type: 'cancelled' as const, + reason: 'notInMappingFile' as const, + }, + }); + } + } + + return result; +} + export function resolveFileNames( fileName: string, getFormatted: (rawName: number | string | undefined) => string | undefined, @@ -479,6 +623,9 @@ export const keyLocalizationMapAttachment = { errorFetchingRecord: attachmentsText.errorFetchingRecord(), saveError: attachmentsText.errorSavingRecord(), attachmentUploadError: attachmentsText.attachmentUploadError(), + fileMissing: attachmentsText.fileMissing(), + notInMappingFile: attachmentsText.notInMappingFile(), + duplicateInMappingFile: attachmentsText.duplicateInMappingFile(), } as const; export function resolveAttachmentStatus( From bdaa197f45983bc88d69daad8df123566bfb1b98 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:05:54 -0500 Subject: [PATCH 04/34] feat(attachments): add MatchingModeDialog and MappingFileSetup components --- .../MappingFileSetup.tsx | 276 ++++++++++++++++++ .../MatchingModeDialog.tsx | 133 +++++++++ 2 files changed, 409 insertions(+) create mode 100644 specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx create mode 100644 specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx new file mode 100644 index 00000000000..c6b896139d8 --- /dev/null +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx @@ -0,0 +1,276 @@ +import React from 'react'; +import { parse } from 'csv-parse/browser/esm'; + +import { attachmentsText } from '../../localization/attachments'; +import { commonText } from '../../localization/common'; +import type { RA } from '../../utils/types'; +import type { GetSet } from '../../utils/types'; +import { filterArray } from '../../utils/types'; +import { Select } from '../Atoms/Form'; +import { FilePicker } from '../Molecules/FilePicker'; +import type { MappingFileColumns, MappingFileRow } from './types'; + +/** + * Parse a CSV file and return headers + data rows + */ +const parseMappingCsv = async ( + file: File +): Promise<{ headers: RA; rows: RA> }> => { + const text = await file.text(); + return new Promise((resolve, reject) => { + parse( + text, + { + delimiter: [',', '\t', '|'], + relaxColumnCount: true, + skipEmptyLines: true, + trim: true, + }, + (error, data: RA>) => { + if (error) { + reject(error); + return; + } + if (data.length === 0) { + reject(new Error('Empty CSV file')); + return; + } + const headers = data[0]; + const rows = data.slice(1).filter( + (row) => row.some((cell) => cell.trim() !== '') + ); + resolve({ headers, rows }); + } + ); + }); +}; + +/** + * Attempt to auto-detect which columns are match value and file name + * by looking at header names. + */ +function autoDetectColumns(headers: RA): { + matchValueIndex: number | undefined; + fileNameIndex: number | undefined; +} { + const matchValuePatterns = [ + /catalog\s*number/i, + /cataloguenumber/i, + /guid/i, + /match\s*value/i, + /identifier/i, + /record\s*id/i, + /barcode/i, + ]; + const fileNamePatterns = [ + /attachment\s*name/i, + /file\s*name/i, + /filename/i, + /attachment/i, + /file/i, + ]; + + let matchValueIndex: number | undefined; + let fileNameIndex: number | undefined; + + headers.forEach((header, index) => { + if ( + matchValueIndex === undefined && + matchValuePatterns.some((p) => p.test(header)) + ) { + matchValueIndex = index; + } + if ( + fileNameIndex === undefined && + fileNamePatterns.some((p) => p.test(header)) + ) { + fileNameIndex = index; + } + }); + + return { matchValueIndex, fileNameIndex }; +} + +export function MappingFileSetup({ + onColumnsSelected: handleColumnsSelected, + disabled, + initialColumns, + initialData, +}: { + readonly onColumnsSelected: ( + columns: MappingFileColumns, + data: RA + ) => void; + readonly disabled?: boolean; + readonly initialColumns?: MappingFileColumns; + readonly initialData?: RA; +}): JSX.Element { + const [file, setFile] = React.useState(undefined); + const [headers, setHeaders] = React.useState | undefined>( + undefined + ); + const [rows, setRows] = React.useState> | undefined>(undefined); + const [matchValueIndex, setMatchValueIndex] = React.useState< + number | undefined + >(initialColumns?.matchValueColumnIndex); + const [fileNameIndex, setFileNameIndex] = React.useState< + number | undefined + >(initialColumns?.fileNameColumnIndex); + const [error, setError] = React.useState(undefined); + + // If initial data is provided (restoring from saved dataset), use it + const isRestored = initialColumns !== undefined && initialData !== undefined; + const [hasRestored] = React.useState(isRestored); + + const handleFileSelected = React.useCallback( + async (selectedFile: File) => { + setFile(selectedFile); + setError(undefined); + try { + const parsed = await parseMappingCsv(selectedFile); + setHeaders(parsed.headers); + setRows(parsed.rows); + + // Auto-detect columns + const detected = autoDetectColumns(parsed.headers); + if (detected.matchValueIndex !== undefined) + setMatchValueIndex(detected.matchValueIndex); + if (detected.fileNameIndex !== undefined) + setFileNameIndex(detected.fileNameIndex); + } catch (err) { + setError( + err instanceof Error ? err.message : 'Failed to parse CSV file' + ); + } + }, + [] + ); + + // Notify parent when columns are selected + const previousMappingRef = React.useRef(undefined); + React.useEffect(() => { + if ( + matchValueIndex === undefined || + fileNameIndex === undefined || + rows === undefined + ) + return; + + const mappingData: RA = filterArray( + rows.map((row) => { + const matchValue = row[matchValueIndex]?.trim(); + const fileName = row[fileNameIndex]?.trim(); + if (matchValue === undefined || fileName === undefined) return undefined; + if (matchValue === '' || fileName === '') return undefined; + return { matchValue, fileName }; + }) + ); + + const mappingKey = JSON.stringify({ matchValueIndex, fileNameIndex }); + if (mappingKey !== previousMappingRef.current) { + previousMappingRef.current = mappingKey; + handleColumnsSelected( + { + matchValueColumnIndex: matchValueIndex, + fileNameColumnIndex: fileNameIndex, + }, + mappingData + ); + } + }, [matchValueIndex, fileNameIndex, rows, handleColumnsSelected]); + + return ( +
+ {/* Show restored state */} + {hasRestored && initialData !== undefined ? ( +
+
+ {attachmentsText.mappingFileLoaded({ + count: initialData.length, + })} +
+
+ {attachmentsText.mappingFileColumnsInfo({ + matchColumn: + headers?.[initialColumns?.matchValueColumnIndex ?? 0] ?? + String(initialColumns?.matchValueColumnIndex ?? 0), + fileColumn: + headers?.[initialColumns?.fileNameColumnIndex ?? 0] ?? + String(initialColumns?.fileNameColumnIndex ?? 0), + })} +
+
+ ) : ( + <> +
+ +
+ {error !== undefined && ( +
{error}
+ )} + + )} + + {/* Column selection */} + {headers !== undefined && ( +
+ + + +
+ )} +
+ ); +} diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx new file mode 100644 index 00000000000..384dac482bf --- /dev/null +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx @@ -0,0 +1,133 @@ +import React from 'react'; + +import { attachmentsText } from '../../localization/attachments'; +import { commonText } from '../../localization/common'; +import type { RA } from '../../utils/types'; +import { Button } from '../Atoms/Button'; +import { Submit } from '../Atoms/Submit'; +import { Dialog } from '../Molecules/Dialog'; +import { MappingFileSetup } from './MappingFileSetup'; +import type { + MappingFileColumns, + MappingFileRow, + MatchingMode, +} from './types'; + +export function MatchingModeDialog({ + onContinue: handleContinue, + onClose: handleClose, + initialMode, + initialColumns, + initialData, +}: { + readonly onContinue: ( + mode: MatchingMode, + columns?: MappingFileColumns, + data?: RA + ) => void; + readonly onClose: () => void; + readonly initialMode?: MatchingMode; + readonly initialColumns?: MappingFileColumns; + readonly initialData?: RA; +}): JSX.Element { + const [mode, setMode] = React.useState( + initialMode ?? 'filename' + ); + const [mappingColumns, setMappingColumns] = React.useState< + MappingFileColumns | undefined + >(initialColumns); + const [mappingData, setMappingData] = React.useState< + RA | undefined + >(initialData); + + const canContinue = + mode === 'filename' || + (mode === 'mappingFile' && + mappingColumns !== undefined && + mappingData !== undefined && + mappingData.length > 0); + + return ( + + {commonText.close()} + + handleContinue(mode, mappingColumns, mappingData) + } + > + {commonText.proceed()} + + + } + header={attachmentsText.matchingMode()} + onClose={handleClose} + > +
+
{attachmentsText.chooseMatchingMode()}
+ + + + +
+ + ); +} From 9f2e3cfc8b97bfe930ea5d2263cda54b5ebf60b8 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:06:02 -0500 Subject: [PATCH 05/34] feat(attachments): integrate mapping-file mode into Import, ViewAttachmentFiles, Datasets, and useEagerDataset --- .../AttachmentsBulkImport/Datasets.tsx | 5 + .../AttachmentsBulkImport/Import.tsx | 271 ++++++++++++++---- .../ViewAttachmentFiles.tsx | 26 +- .../AttachmentsBulkImport/useEagerDataset.ts | 9 +- 4 files changed, 252 insertions(+), 59 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Datasets.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Datasets.tsx index fda1cfda79d..cb774fb52e2 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Datasets.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Datasets.tsx @@ -208,6 +208,11 @@ export function AttachmentsImportOverlay(): JSX.Element | null { /> {attachmentDataSet.name} + {attachmentDataSet.matchingmode === 'mappingFile' && ( +
+ {attachmentsText.matchByMappingFile()} +
+ )} diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx index 7bd0afe0fbd..57cf24e6291 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx @@ -30,19 +30,25 @@ import { SelectUploadPath } from './SelectUploadPath'; import type { AttachmentDataSet, FetchedDataSet, + MappingFileColumns, + MappingFileRow, + MatchingMode, PartialUploadableFileSpec, UnBoundFile, } from './types'; import { AttachmentUpload } from './Upload'; import { useEagerDataSet } from './useEagerDataset'; import { + crossReferenceMappingFiles, matchSelectedFiles, + prepareMappingFileSelection, reconstructDeletingAttachment, reconstructUploadingAttachmentSpec, resolveFileNames, } from './utils'; import { AttachmentsValidationDialog } from './ValidationDialog'; import { ViewAttachmentFiles } from './ViewAttachmentFiles'; +import { MatchingModeDialog } from './MatchingModeDialog'; export type AttachmentUploadSpec = { readonly staticPathKey: keyof typeof staticAttachmentImportPaths; @@ -52,6 +58,9 @@ export type AttachmentUploadSpec = { }; export type PartialAttachmentUploadSpec = { readonly fieldFormatter?: UiFormatter; + readonly matchingMode?: MatchingMode; + readonly mappingFileColumns?: MappingFileColumns; + readonly mappingFileData?: RA; } & (AttachmentUploadSpec | { readonly staticPathKey: undefined }); export type EagerDataSet = Omit & { @@ -132,41 +141,113 @@ function AttachmentsImport({ ); const applyFileNames = React.useCallback( - (file: UnBoundFile): PartialUploadableFileSpec => - eagerDataSet.uploadplan.staticPathKey === undefined - ? { uploadFile: file } - : { - uploadFile: { - ...file, - parsedName: resolveFileNames( - file.file.name, - eagerDataSet.uploadplan.formatQueryResults, - eagerDataSet.uploadplan.fieldFormatter - ), - }, + (file: UnBoundFile): PartialUploadableFileSpec => { + // In mapping-file mode, look up the match value from the CSV data + // (works even before a field path is selected) + if ( + eagerDataSet.uploadplan.matchingMode === 'mappingFile' && + eagerDataSet.uploadplan.mappingFileData !== undefined + ) { + const mappingRow = eagerDataSet.uploadplan.mappingFileData.find( + (row) => row.fileName === file.file.name + ); + return { + uploadFile: { + ...file, + parsedName: mappingRow?.matchValue, + mappingMatchValue: mappingRow?.matchValue, + mappingFileName: mappingRow?.fileName, }, - [eagerDataSet.uploadplan.staticPathKey] + }; + } + + if (eagerDataSet.uploadplan.staticPathKey === undefined) + return { uploadFile: file }; + + // Default: filename pattern matching + return { + uploadFile: { + ...file, + parsedName: resolveFileNames( + file.file.name, + eagerDataSet.uploadplan.formatQueryResults, + eagerDataSet.uploadplan.fieldFormatter + ), + }, + }; + }, + [ + eagerDataSet.uploadplan.staticPathKey, + eagerDataSet.uploadplan.matchingMode, + eagerDataSet.uploadplan.mappingFileData, + eagerDataSet.uploadplan.formatQueryResults, + eagerDataSet.uploadplan.fieldFormatter, + ] ); const previousKeyRef = React.useRef( - attachmentDataSetResource.uploadplan.staticPathKey + `${attachmentDataSetResource.uploadplan.staticPathKey ?? ''}_${attachmentDataSetResource.uploadplan.matchingMode ?? 'filename'}` ); React.useEffect(() => { - // Reset all parsed names if matching path is changed - if (previousKeyRef.current !== eagerDataSet.uploadplan.staticPathKey) { - previousKeyRef.current = eagerDataSet.uploadplan.staticPathKey; - commitFileChange((files) => - files.map(({ uploadFile }) => applyFileNames(uploadFile)) - ); + // Reset all parsed names if matching path or mode is changed + const currentKey = `${eagerDataSet.uploadplan.staticPathKey ?? ''}_${eagerDataSet.uploadplan.matchingMode ?? 'filename'}`; + if (previousKeyRef.current !== currentKey) { + previousKeyRef.current = currentKey; + commitFileChange((files) => { + const recalculated = files.map(({ uploadFile }) => + applyFileNames(uploadFile) + ); + // If switching away from mapping mode, remove placeholder entries + if (!isMappingMode) { + return recalculated.filter( + (f) => + !( + f.status?.type === 'cancelled' && + f.status.reason === 'fileMissing' + ) + ); + } + return recalculated; + }); } }, [applyFileNames, commitFileChange]); + // In mapping mode, seed the table with CSV rows so the user sees what to upload + React.useEffect(() => { + if ( + isMappingMode && + eagerDataSet.uploadplan.mappingFileData !== undefined && + eagerDataSet.uploadplan.mappingFileData.length > 0 && + eagerDataSet.rows.length === 0 + ) { + commitFileChange(() => + crossReferenceMappingFiles( + [], + eagerDataSet.uploadplan.mappingFileData! + ) + ); + } + }, [ + eagerDataSet.uploadplan.matchingMode, + eagerDataSet.uploadplan.mappingFileData, + eagerDataSet.rows.length, + commitFileChange, + ]); + const currentBaseTable = eagerDataSet.uploadplan.staticPathKey === undefined ? undefined : staticAttachmentImportPaths[eagerDataSet.uploadplan.staticPathKey] .baseTable; + // Single source of truth: dataset-level matchingmode (DB field) or uploadplan-level + const isMappingMode = React.useMemo( + () => + (eagerDataSet.matchingmode ?? eagerDataSet.uploadplan.matchingMode) === + 'mappingFile', + [eagerDataSet.matchingmode, eagerDataSet.uploadplan.matchingMode] + ); + const anyUploaded = React.useMemo( () => eagerDataSet.rows.some( @@ -177,18 +258,31 @@ function AttachmentsImport({ const handleFilesSelected = (files: FileList) => { const filesList = Array.from(files, (file) => applyFileNames({ file })); - const oldRows = eagerDataSet.rows; - const { resolvedFiles, duplicateFiles } = matchSelectedFiles( - oldRows, - filesList - ); - (resolvedFiles as WritableArray).sort( - sortFunction((file) => file.uploadFile.file.name) + const { resolvedFiles, duplicateFiles } = + isMappingMode && eagerDataSet.uploadplan.mappingFileData !== undefined + ? prepareMappingFileSelection( + eagerDataSet.rows, + filesList, + eagerDataSet.uploadplan.mappingFileData + ) + : matchSelectedFiles(eagerDataSet.rows, filesList); + + // Safety net: guarantee no duplicate filenames in the final list + const seen = new Set(); + const deduped = (resolvedFiles as WritableArray).filter( + (f) => { + const name = f.uploadFile.file.name; + if (seen.has(name)) return false; + seen.add(name); + return true; + } ); + + deduped.sort(sortFunction((file) => file.uploadFile.file.name)); commitChange((oldState) => ({ ...oldState, uploaderstatus: 'main', - rows: resolvedFiles, + rows: deduped, })); setDuplicatedFiles(duplicateFiles); }; @@ -200,33 +294,60 @@ function AttachmentsImport({ >([]); const mainHeaders = React.useMemo(() => { - let headers: IR = { - selectedFileName: commonText.selectedFileName(), - fileSize: attachmentsText.fileSize(), - record: ( -
- {currentBaseTable === undefined ? ( - userText.resource() - ) : ( - <> - - {eagerDataSet.uploadplan.staticPathKey === undefined - ? '' - : strictGetTable(currentBaseTable).strictGetField( - staticAttachmentImportPaths[ - eagerDataSet.uploadplan.staticPathKey - ].path - ).label} - - )} -
- ), - progress: attachmentsText.progress(), - }; + const isMapping = isMappingMode; + const baseHeaders: IR = isMapping + ? { + selectedFileName: commonText.selectedFileName(), + matchValue: attachmentsText.matchValue(), + fileSize: attachmentsText.fileSize(), + record: ( +
+ {currentBaseTable === undefined ? ( + userText.resource() + ) : ( + <> + + {eagerDataSet.uploadplan.staticPathKey === undefined + ? '' + : strictGetTable(currentBaseTable).strictGetField( + staticAttachmentImportPaths[ + eagerDataSet.uploadplan.staticPathKey + ].path + ).label} + + )} +
+ ), + progress: attachmentsText.progress(), + } + : { + selectedFileName: commonText.selectedFileName(), + fileSize: attachmentsText.fileSize(), + record: ( +
+ {currentBaseTable === undefined ? ( + userText.resource() + ) : ( + <> + + {eagerDataSet.uploadplan.staticPathKey === undefined + ? '' + : strictGetTable(currentBaseTable).strictGetField( + staticAttachmentImportPaths[ + eagerDataSet.uploadplan.staticPathKey + ].path + ).label} + + )} +
+ ), + progress: attachmentsText.progress(), + }; + let headers = baseHeaders; if (process.env.NODE_ENV === 'development') headers = { ...headers, attachmentId: attachmentsText.attachmentId() }; return headers; - }, [eagerDataSet.uploadplan.staticPathKey]); + }, [eagerDataSet.uploadplan.staticPathKey, eagerDataSet.uploadplan.matchingMode]); const errorContextData = React.useMemo( () => ({ @@ -238,6 +359,11 @@ function AttachmentsImport({ useErrorContext('bulkAttachmentImport', errorContextData); + const showModeDialog = + (eagerDataSet.matchingmode === undefined || + eagerDataSet.matchingmode === null) && + eagerDataSet.uploadplan.staticPathKey === undefined; + return (
@@ -269,7 +395,14 @@ function AttachmentsImport({ : (uploadSpec) => { commitChange((oldState) => ({ ...oldState, - uploadplan: uploadSpec, + uploadplan: { + ...uploadSpec, + matchingMode: oldState.uploadplan.matchingMode, + mappingFileColumns: + oldState.uploadplan.mappingFileColumns, + mappingFileData: + oldState.uploadplan.mappingFileData, + }, })); } } @@ -330,6 +463,7 @@ function AttachmentsImport({ @@ -422,6 +556,7 @@ function AttachmentsImport({ )} + {showModeDialog && ( + { + // Default to filename mode if user closes without choosing + commitChange((oldState) => ({ + ...oldState, + matchingmode: 'filename' as MatchingMode, + uploadplan: { + ...oldState.uploadplan, + matchingMode: 'filename' as MatchingMode, + }, + })); + }} + onContinue={(mode, columns, data) => { + commitChange((oldState) => ({ + ...oldState, + matchingmode: mode, + uploadplan: { + ...oldState.uploadplan, + matchingMode: mode, + mappingFileColumns: columns, + mappingFileData: data, + }, + })); + }} + /> + )} ); } diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx index 276b19add08..a16214b2d93 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx @@ -27,7 +27,8 @@ import { const resolveAttachmentDatasetData = ( uploadableFiles: RA, setDisambiguationIndex: (index: number) => void, - baseTableName: keyof Tables | undefined + baseTableName: keyof Tables | undefined, + isMappingMode: boolean = false ) => uploadableFiles.map( ({ uploadFile, status, matchedId, disambiguated, attachmentId }, index) => { @@ -46,7 +47,8 @@ const resolveAttachmentDatasetData = ( : resolveAttachmentRecord( matchedId, disambiguated, - uploadFile.parsedName + uploadFile.parsedName, + isMappingMode ); const isRuntimeError = @@ -55,7 +57,8 @@ const resolveAttachmentDatasetData = ( (status.type === 'cancelled' || status.type === 'skipped'); const statusText = f.maybe(status, resolveAttachmentStatus) ?? ''; - return { + + const baseData = { selectedFileName: [ uploadFile.file.name,
@@ -100,6 +103,16 @@ const resolveAttachmentDatasetData = ( isNativeError: resolvedRecord?.type === 'invalid', isRuntimeError, } as const; + + return isMappingMode + ? { + ...baseData, + matchValue: [ + uploadFile.mappingMatchValue ?? '', + {uploadFile.mappingMatchValue ?? ''}, + ] as const, + } + : baseData; } ); @@ -109,6 +122,7 @@ export function ViewAttachmentFiles({ onDisambiguation: handleDisambiguation, onFilesDropped: handleFilesDropped, headers, + isMappingMode = false, }: { readonly uploadableFiles: RA; readonly baseTableName: keyof Tables | undefined; @@ -122,6 +136,7 @@ export function ViewAttachmentFiles({ | undefined; readonly onFilesDropped?: (file: FileList) => void; readonly headers: IR; + readonly isMappingMode?: boolean; }): JSX.Element | null { const [disambiguationIndex, setDisambiguationIndex] = React.useState< number | undefined @@ -132,9 +147,10 @@ export function ViewAttachmentFiles({ resolveAttachmentDatasetData( uploadableFiles, setDisambiguationIndex, - baseTableName + baseTableName, + isMappingMode ), - [uploadableFiles, setDisambiguationIndex, baseTableName] + [uploadableFiles, setDisambiguationIndex, baseTableName, isMappingMode] ); const fileDropDivRef = React.useRef(null); diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/useEagerDataset.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/useEagerDataset.ts index 437bb9c25dc..09985bbf6fb 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/useEagerDataset.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/useEagerDataset.ts @@ -89,7 +89,14 @@ export function useEagerDataSet(baseDataSet: AttachmentDataSet): { needsSaved: baseDataSet.uploaderstatus !== 'main', rows: baseDataSet.rows ?? [], save: false, - uploadplan: generateUploadSpec(baseDataSet.uploadplan.staticPathKey), + uploadplan: { + ...generateUploadSpec(baseDataSet.uploadplan.staticPathKey), + matchingMode: + baseDataSet.matchingmode ?? + baseDataSet.uploadplan.matchingMode, + mappingFileColumns: baseDataSet.uploadplan.mappingFileColumns, + mappingFileData: baseDataSet.uploadplan.mappingFileData, + }, uploadresult: baseDataSet.uploadresult === undefined || baseDataSet.uploadresult === null From d912787fb054cec3939a0180d438a29d66c6c90b Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:08:21 -0500 Subject: [PATCH 06/34] chore: improve descriptions --- .../js_src/lib/localization/attachments.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/localization/attachments.ts b/specifyweb/frontend/js_src/lib/localization/attachments.ts index 48bb7fde364..33606a7f9f8 100644 --- a/specifyweb/frontend/js_src/lib/localization/attachments.ts +++ b/specifyweb/frontend/js_src/lib/localization/attachments.ts @@ -1042,7 +1042,7 @@ export const attachmentsText = createDictionary({ nb: 'Velg hvordan vedleggsfiler skal matches mot databaseposter.', }, matchByFilenameDescription: { - 'en-us': 'Parse the record identifier from each filename using a field formatter. Works best with structured, numeric fields like Catalog Number.', + 'en-us': 'Parse the record identifier from each filename using a field formatter. Works best with structured numeric fields.', 'de-ch': 'Analysieren Sie die Datensatzkennung aus jedem Dateinamen mit einem Feldformatierer. Funktioniert am besten mit strukturierten, numerischen Feldern.', 'es-es': 'Analice el identificador de registro de cada nombre de archivo utilizando un formateador de campo. Funciona mejor con campos numéricos estructurados.', 'fr-fr': "Analysez l'identifiant d'enregistrement à partir de chaque nom de fichier à l'aide d'un formateur de champ. Fonctionne mieux avec des champs numériques structurés.", @@ -1053,15 +1053,15 @@ export const attachmentsText = createDictionary({ nb: 'Analyser postidentifikatoren fra hvert filnavn ved hjelp av en feltformaterer. Fungerer best med strukturerte, numeriske felt.', }, matchByMappingFileDescription: { - 'en-us': 'Use an explicit CSV mapping file that lists which record each file belongs to. No filename parsing required — works with any field format, including free text.', - 'de-ch': 'Verwenden Sie eine explizite CSV-Zuordnungsdatei, die auflistet, zu welchem Datensatz jede Datei gehört. Keine Dateinamenanalyse erforderlich.', - 'es-es': 'Use un archivo de mapeo CSV explícito que indique a qué registro pertenece cada archivo. No se requiere análisis de nombres de archivo.', - 'fr-fr': "Utilisez un fichier CSV de mappage explicite qui répertorie à quel enregistrement appartient chaque fichier. Aucune analyse du nom de fichier requise.", - 'ru-ru': 'Используйте явный CSV-файл сопоставления, в котором указано, к какой записи относится каждый файл. Анализ имени файла не требуется.', - 'uk-ua': 'Використовуйте явний CSV-файл зіставлення, у якому вказано, до якого запису належить кожен файл. Аналіз назви файлу не потрібен.', - 'pt-br': 'Use um arquivo CSV de mapeamento explícito que lista a qual registro cada arquivo pertence. Nenhuma análise de nome de arquivo é necessária.', - 'hr-hr': 'Koristite eksplicitnu CSV datoteku mapiranja koja navodi kojem zapisu pripada svaka datoteka. Nije potrebna analiza naziva datoteke.', - nb: 'Bruk en eksplisitt CSV-tilordningsfil som viser hvilken post hver fil tilhører. Ingen filnavnanalyse kreves.', + 'en-us': 'Use an explicit CSV mapping file that lists which record each file belongs to.', + 'de-ch': 'Verwenden Sie eine explizite CSV-Zuordnungsdatei, die angibt, zu welchem Datensatz jede Datei gehört.', + 'es-es': 'Utilice un archivo de mapeo CSV explícito que indique a qué registro pertenece cada archivo.', + 'fr-fr': "Utilisez un fichier de mappage CSV explicite qui indique à quel enregistrement appartient chaque fichier.", + 'ru-ru': 'Используйте явный CSV-файл сопоставления, который указывает, к какой записи принадлежит каждый файл.', + 'uk-ua': 'Використовуйте явний CSV-файл зіставлення, який вказує, до якого запису належить кожен файл.', + 'pt-br': 'Use um arquivo de mapeamento CSV explícito que liste a qual registro cada arquivo pertence.', + 'hr-hr': 'Koristite eksplicitnu CSV datoteku mapiranja koja navodi kojem zapisu pripada svaka datoteka.', + nb: 'Bruk en eksplisitt CSV-tilordningsfil som viser hvilken post hver fil tilhører.', }, matchValue: { 'en-us': 'Match Value', From e9b54a606d347dd9d87d962e3896ebe294143284 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:20:22 -0500 Subject: [PATCH 07/34] feat(attachments): match CSV columns against schema field names --- .../MappingFileSetup.tsx | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx index c6b896139d8..d2dbcbbfee0 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx @@ -4,10 +4,12 @@ import { parse } from 'csv-parse/browser/esm'; import { attachmentsText } from '../../localization/attachments'; import { commonText } from '../../localization/common'; import type { RA } from '../../utils/types'; -import type { GetSet } from '../../utils/types'; import { filterArray } from '../../utils/types'; +import { escapeRegExp } from '../../utils/utils'; import { Select } from '../Atoms/Form'; import { FilePicker } from '../Molecules/FilePicker'; +import { strictGetTable } from '../DataModel/tables'; +import { staticAttachmentImportPaths } from './importPaths'; import type { MappingFileColumns, MappingFileRow } from './types'; /** @@ -46,22 +48,53 @@ const parseMappingCsv = async ( }; /** - * Attempt to auto-detect which columns are match value and file name - * by looking at header names. + * Build case-insensitive regex matchers for attachment field names. + * Matches against both the raw field name (e.g. "catalogNumber") and + * the schema-localized caption (e.g. "Specimen #", "Catalog Number"). + */ +function buildFieldMatchers(): RA { + const matchers: RegExp[] = []; + const seen = new Set(); + + for (const { baseTable, path } of Object.values( + staticAttachmentImportPaths + )) { + const escaped = escapeRegExp(String(path)); + const key = escaped.toLowerCase(); + if (!seen.has(key)) { + seen.add(key); + matchers.push(new RegExp(`^${escaped}$`, 'i')); + } + + // Also match against the schema-localized field caption + try { + const field = strictGetTable(baseTable).strictGetField(path); + const label = String(field.label); + const escapedLabel = escapeRegExp(label); + const labelKey = escapedLabel.toLowerCase(); + if (label !== String(path) && !seen.has(labelKey)) { + seen.add(labelKey); + matchers.push(new RegExp(`^${escapedLabel}$`, 'i')); + } + } catch { + // Schema not yet loaded — skip localized labels + } + } + + return matchers; +} + +/** + * Attempt to auto-detect which columns are match value and file name. + * Match value columns are detected by comparing CSV headers against + * data model field names and schema-localized field captions. */ function autoDetectColumns(headers: RA): { matchValueIndex: number | undefined; fileNameIndex: number | undefined; } { - const matchValuePatterns = [ - /catalog\s*number/i, - /cataloguenumber/i, - /guid/i, - /match\s*value/i, - /identifier/i, - /record\s*id/i, - /barcode/i, - ]; + const fieldMatchers = buildFieldMatchers(); + const fileNamePatterns = [ /attachment\s*name/i, /file\s*name/i, @@ -76,7 +109,7 @@ function autoDetectColumns(headers: RA): { headers.forEach((header, index) => { if ( matchValueIndex === undefined && - matchValuePatterns.some((p) => p.test(header)) + fieldMatchers.some((p) => p.test(header)) ) { matchValueIndex = index; } @@ -105,7 +138,6 @@ export function MappingFileSetup({ readonly initialColumns?: MappingFileColumns; readonly initialData?: RA; }): JSX.Element { - const [file, setFile] = React.useState(undefined); const [headers, setHeaders] = React.useState | undefined>( undefined ); @@ -124,7 +156,6 @@ export function MappingFileSetup({ const handleFileSelected = React.useCallback( async (selectedFile: File) => { - setFile(selectedFile); setError(undefined); try { const parsed = await parseMappingCsv(selectedFile); From 6e00a7041f371aae8b1bc977292781766341d8df Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:21:08 -0500 Subject: [PATCH 08/34] feat(attachments): add mapping file tests --- .../__tests__/utils.test.ts | 424 ++++++++++++++++++ 1 file changed, 424 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts index 1b7aeb3da31..d93b8b8f926 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/__tests__/utils.test.ts @@ -16,7 +16,9 @@ import { crossReferenceMappingFiles, inferDeletedAttachments, inferUploadedAttachments, + isMappingFilePlaceholder, matchFileSpec, + matchSelectedFiles, prepareMappingFileSelection, resolveFileNames, } from '../utils'; @@ -745,3 +747,425 @@ describe('crossReferenceMappingFiles CSV always wins matchValue', () => { expect(result[0].uploadFile.mappingFileName).toBe('test.jpg'); }); }); + +// --------------------------------------------------------------------------- +// isMappingFilePlaceholder +// --------------------------------------------------------------------------- +describe('isMappingFilePlaceholder', () => { + test('returns true for fileMissing cancelled status', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 0, type: '' } }, + status: { type: 'cancelled', reason: 'fileMissing' }, + } as PartialUploadableFileSpec) + ).toBe(true); + }); + + test('returns false for other cancelled reasons', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 0, type: '' } }, + status: { type: 'cancelled', reason: 'noMatch' }, + } as PartialUploadableFileSpec) + ).toBe(false); + }); + + test('returns false for skipped status', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 0, type: '' } }, + status: { type: 'skipped', reason: 'fileMissing' }, + } as PartialUploadableFileSpec) + ).toBe(false); + }); + + test('returns false for success status', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 0, type: '' } }, + status: { type: 'success', successType: 'uploaded' }, + } as PartialUploadableFileSpec) + ).toBe(false); + }); + + test('returns false when status is undefined', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 91, type: 'image/jpeg' } }, + } as PartialUploadableFileSpec) + ).toBe(false); + }); + + test('returns false for matched status', () => { + expect( + isMappingFilePlaceholder({ + uploadFile: { file: { name: 'a.jpg', size: 91, type: 'image/jpeg' } }, + status: { type: 'matched', id: 1 }, + } as PartialUploadableFileSpec) + ).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// matchSelectedFiles – existing function, now tested with mapping scenarios +// --------------------------------------------------------------------------- +describe('matchSelectedFiles', () => { + const mk = (name: string, size = 91) => + ({ + uploadFile: { + file: { name, size, type: 'image/jpeg' }, + }, + }) as PartialUploadableFileSpec; + + test('adds new files to empty list', () => { + const { resolvedFiles, duplicateFiles } = matchSelectedFiles( + [], + [mk('a.jpg'), mk('b.jpg')] + ); + expect(resolvedFiles).toHaveLength(2); + expect(duplicateFiles).toHaveLength(0); + }); + + test('detects exact duplicate when existing entry has File instance', () => { + const content = new Uint8Array(91); + const file = new File([content], 'a.jpg', { type: 'image/jpeg' }); + const existing = [ + { uploadFile: { file } }, + ] as RA; + const { resolvedFiles, duplicateFiles } = matchSelectedFiles(existing, [ + mk('a.jpg'), + ]); + expect(resolvedFiles).toHaveLength(1); + expect(duplicateFiles).toHaveLength(1); + expect(duplicateFiles[0].uploadFile.file.name).toBe('a.jpg'); + }); + + test('placeholder (0 bytes) does NOT match real file (91 bytes) — both kept', () => { + const placeholder = mk('a.jpg', 0); + const real = mk('a.jpg', 91); + const { resolvedFiles } = matchSelectedFiles([placeholder], [real]); + expect(resolvedFiles).toHaveLength(2); + }); + + test('replaces non-File existing entry with new entry', () => { + // Simulating a serialized row (plain object, not File instance) + const saved = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' }, + }, + } as PartialUploadableFileSpec; + const real = mk('a.jpg', 91); + const { resolvedFiles, duplicateFiles } = matchSelectedFiles( + [saved], + [real] + ); + expect(resolvedFiles).toHaveLength(1); + expect(duplicateFiles).toHaveLength(0); + // The new entry's uploadFile replaced the old one + expect(resolvedFiles[0].uploadFile).toBe(real.uploadFile); + }); + + test('preserves success status when replacing', () => { + const saved = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' } as File, + }, + status: { type: 'success', successType: 'uploaded' }, + } as PartialUploadableFileSpec; + const real = mk('a.jpg', 91); + const { resolvedFiles } = matchSelectedFiles([saved], [real]); + expect(resolvedFiles).toHaveLength(1); + expect(resolvedFiles[0].status?.type).toBe('success'); + }); + + test('preserves skipped/alreadyUploaded status when replacing', () => { + const saved = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' } as File, + }, + status: { type: 'skipped', reason: 'alreadyUploaded' }, + } as PartialUploadableFileSpec; + const real = mk('a.jpg', 91); + const { resolvedFiles } = matchSelectedFiles([saved], [real]); + expect(resolvedFiles).toHaveLength(1); + expect(resolvedFiles[0].status?.type).toBe('skipped'); + }); + + test('takes new cancelled status when replacing non-success entry', () => { + const saved = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' } as File, + }, + status: { type: 'cancelled', reason: 'noMatch' }, + } as PartialUploadableFileSpec; + const real = { + ...mk('a.jpg', 91), + status: { type: 'cancelled', reason: 'fileMissing' }, + } as PartialUploadableFileSpec; + const { resolvedFiles } = matchSelectedFiles([saved], [real]); + expect(resolvedFiles).toHaveLength(1); + expect(resolvedFiles[0].status?.reason).toBe('fileMissing'); + }); +}); + +// --------------------------------------------------------------------------- +// crossReferenceMappingFiles – additional edge cases +// --------------------------------------------------------------------------- +describe('crossReferenceMappingFiles – edge cases', () => { + const mk = ( + name: string, + size: number, + mv?: string + ): PartialUploadableFileSpec => + ({ + uploadFile: { + file: { name, size, type: size > 0 ? 'image/jpeg' : '' }, + parsedName: mv, + mappingMatchValue: mv, + mappingFileName: mv !== undefined ? name : undefined, + }, + }) as PartialUploadableFileSpec; + + test('empty CSV returns empty result', () => { + expect(crossReferenceMappingFiles([], [])).toHaveLength(0); + }); + + test('empty uploadables with CSV produces all placeholders', () => { + const csv = [ + { fileName: 'a.jpg', matchValue: '1' }, + { fileName: 'b.jpg', matchValue: '2' }, + ]; + const result = crossReferenceMappingFiles([], csv); + expect(result).toHaveLength(2); + expect(result.every((r) => r.status?.reason === 'fileMissing')).toBe(true); + expect(result[0].uploadFile.mappingMatchValue).toBe('1'); + expect(result[1].uploadFile.mappingMatchValue).toBe('2'); + }); + + test('uploadables not in CSV are flagged notInMappingFile', () => { + const csv = [{ fileName: 'a.jpg', matchValue: '1' }]; + const result = crossReferenceMappingFiles( + [mk('a.jpg', 91, '1'), mk('b.jpg', 91), mk('c.jpg', 91)], + csv + ); + const b = result.find((r) => r.uploadFile.file.name === 'b.jpg'); + const c = result.find((r) => r.uploadFile.file.name === 'c.jpg'); + expect(b?.status?.reason).toBe('notInMappingFile'); + expect(c?.status?.reason).toBe('notInMappingFile'); + // 'a' should be present and matched + expect( + result.find((r) => r.uploadFile.file.name === 'a.jpg')?.status + ).toBeUndefined(); + }); + + test('already-uploaded file (with attachmentId) is preserved through cross-reference', () => { + const csv = [{ fileName: 'a.jpg', matchValue: '1' }]; + const uploaded = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' }, + parsedName: '1', + mappingMatchValue: '1', + }, + attachmentId: 42, + status: { type: 'success', successType: 'uploaded' }, + } as PartialUploadableFileSpec; + + const result = crossReferenceMappingFiles([uploaded], csv); + expect(result).toHaveLength(1); + expect(result[0].attachmentId).toBe(42); + expect(result[0].status?.type).toBe('success'); + }); + + test('byName prefers real File (size 91) over plain object (size 91)', () => { + const csv = [{ fileName: 'a.jpg', matchValue: '1' }]; + // Plain object first (would be first in byName), then real file + const plainObj = { + uploadFile: { + file: { name: 'a.jpg', size: 91, type: 'image/jpeg' }, + parsedName: '1', + mappingMatchValue: 'wrong', + }, + } as PartialUploadableFileSpec; + const realFile = mk('a.jpg', 91, '1'); + // Give realFile a distinguishable property — it has mappingMatchValue set correctly + const result = crossReferenceMappingFiles([plainObj, realFile], csv); + expect(result).toHaveLength(1); + // mappingMatchValue from CSV overrides regardless — both should be '1' + expect(result[0].uploadFile.mappingMatchValue).toBe('1'); + }); + + test('multiple CSV rows with same filename produce exactly one output row', () => { + const csv = [ + { fileName: 'a.jpg', matchValue: '1' }, + { fileName: 'a.jpg', matchValue: '1' }, + { fileName: 'b.jpg', matchValue: '2' }, + ]; + const result = crossReferenceMappingFiles([], csv); + expect(result).toHaveLength(2); + const a = result.filter((r) => r.uploadFile.file.name === 'a.jpg'); + expect(a).toHaveLength(1); + }); + + test('triple duplicate filename in CSV produces one row', () => { + const csv = [ + { fileName: 'x.jpg', matchValue: 'A' }, + { fileName: 'x.jpg', matchValue: 'A' }, + { fileName: 'x.jpg', matchValue: 'A' }, + ]; + const result = crossReferenceMappingFiles([mk('x.jpg', 91, 'A')], csv); + expect(result).toHaveLength(1); + }); + + test('placeholder entries have correct parsedName and mappingMatchValue from CSV', () => { + const csv = [{ fileName: 'test.jpg', matchValue: 'ABC-123' }]; + const result = crossReferenceMappingFiles([], csv); + expect(result).toHaveLength(1); + expect(result[0].uploadFile.parsedName).toBe('ABC-123'); + expect(result[0].uploadFile.mappingMatchValue).toBe('ABC-123'); + expect(result[0].uploadFile.file.size).toBe(0); + }); + + test('status is cleared when real file replaces placeholder (no error status)', () => { + const csv = [{ fileName: 'a.jpg', matchValue: '1' }]; + const seeded = crossReferenceMappingFiles([], csv); + expect(seeded[0].status?.reason).toBe('fileMissing'); + + const result = crossReferenceMappingFiles( + [...seeded, mk('a.jpg', 91, '1')], + csv + ); + expect(result).toHaveLength(1); + expect(result[0].status).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// prepareMappingFileSelection – additional edge cases +// --------------------------------------------------------------------------- +describe('prepareMappingFileSelection – edge cases', () => { + const CSV = [ + { matchValue: 'A', fileName: 'a.jpg' }, + { matchValue: 'B', fileName: 'b.jpg' }, + ]; + + const mk = (name: string, size: number, mv?: string) => + ({ + uploadFile: { + file: { name, size, type: size > 0 ? 'image/jpeg' : '' }, + parsedName: mv, + mappingMatchValue: mv, + mappingFileName: mv !== undefined ? name : undefined, + }, + ...(size === 0 + ? { status: { type: 'cancelled' as const, reason: 'fileMissing' as const } } + : {}), + }) as PartialUploadableFileSpec; + + test('empty inputs produce only placeholders from CSV', () => { + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + [], + [], + CSV + ); + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(2); + expect(resolvedFiles.every((r) => r.status?.reason === 'fileMissing')).toBe( + true + ); + }); + + test('existing real files not in new selection are preserved', () => { + const existing = [mk('a.jpg', 91, 'A')]; + const { resolvedFiles } = prepareMappingFileSelection( + existing, + [mk('b.jpg', 91, 'B')], + CSV + ); + expect(resolvedFiles).toHaveLength(2); + expect( + resolvedFiles.find((r) => r.uploadFile.file.name === 'a.jpg') + ).toBeTruthy(); + expect( + resolvedFiles.find((r) => r.uploadFile.file.name === 'b.jpg') + ).toBeTruthy(); + }); + + test('existing placeholders are NOT preserved', () => { + const placeholders = [ + mk('a.jpg', 0, 'A'), + mk('b.jpg', 0, 'B'), + ]; + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + placeholders, + [mk('a.jpg', 91, 'A')], + CSV + ); + expect(duplicateFiles).toHaveLength(0); + expect(resolvedFiles).toHaveLength(2); + // a.jpg should now be real (91 bytes), b.jpg should be a placeholder + const a = resolvedFiles.find((r) => r.uploadFile.file.name === 'a.jpg'); + const b = resolvedFiles.find((r) => r.uploadFile.file.name === 'b.jpg'); + expect(a?.uploadFile.file.size).toBe(91); + expect(b?.status?.reason).toBe('fileMissing'); + }); + + test('duplicate files (same name/size/type) are detected', () => { + // Create a real File with 91 bytes so it matches the new entry + const content = new Uint8Array(91); + const existingFile = new File([content], 'a.jpg', { type: 'image/jpeg' }); + const existing = [ + { + uploadFile: { file: existingFile, parsedName: 'A', mappingMatchValue: 'A' }, + }, + ] as RA; + const { resolvedFiles, duplicateFiles } = prepareMappingFileSelection( + existing, + [mk('a.jpg', 91, 'A')], + CSV + ); + expect(duplicateFiles).toHaveLength(1); + expect(resolvedFiles).toHaveLength(2); // a (existing) + b (placeholder) + }); + + test('files with attachmentId are preserved through selection', () => { + const existing = [ + { + ...mk('a.jpg', 91, 'A'), + attachmentId: 99, + status: { type: 'success' as const, successType: 'uploaded' as const }, + }, + ] as RA; + + const { resolvedFiles } = prepareMappingFileSelection( + existing, + [mk('b.jpg', 91, 'B')], + CSV + ); + const a = resolvedFiles.find((r) => r.uploadFile.file.name === 'a.jpg'); + expect(a?.attachmentId).toBe(99); + expect(a?.status?.type).toBe('success'); + }); + + test('old rows without mappingMatchValue get it from CSV after selection', () => { + const savedRows = CSV.map( + (row) => + ({ + uploadFile: { + file: { name: row.fileName, size: 91, type: 'image/jpeg' }, + parsedName: row.matchValue, + }, + }) as PartialUploadableFileSpec + ); + + const { resolvedFiles } = prepareMappingFileSelection( + savedRows, + [], + CSV + ); + + for (const r of resolvedFiles) { + expect(r.uploadFile.mappingMatchValue).toBeTruthy(); + expect(r.uploadFile.parsedName).toBeTruthy(); + } + }); +}); From 94567616f1a266a0a33f474d6bae2c5525231518 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:26:07 -0500 Subject: [PATCH 09/34] fix(attachments): remove spurious canceled for duplicates --- .../components/AttachmentsBulkImport/utils.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index f7cc3a51885..703393cbc9e 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -293,22 +293,12 @@ export function crossReferenceMappingFiles( } } - // Count duplicates in the CSV for flagging - const mappingFileNameCounts = new Map(); - for (const row of mappingData) { - mappingFileNameCounts.set( - row.fileName, - (mappingFileNameCounts.get(row.fileName) ?? 0) + 1 - ); - } - const result: RA = []; const matchedFileNames = new Set(); // For each CSV row, emit the best available file or a placeholder for (const row of mappingData) { const existing = byName.get(row.fileName); - const isDuplicateInCsv = (mappingFileNameCounts.get(row.fileName) ?? 0) > 1; if (existing !== undefined) { // Only emit once per filename even if CSV has duplicate rows @@ -323,14 +313,6 @@ export function crossReferenceMappingFiles( mappingMatchValue: row.matchValue, mappingFileName: row.fileName, }, - ...(isDuplicateInCsv && existing.attachmentId === undefined - ? { - status: { - type: 'cancelled' as const, - reason: 'duplicateInMappingFile' as const, - }, - } - : {}), }); } else { // File not yet uploaded — add placeholder (once per filename) From da57dc62e1baee6c4e37c395838e6c9f75a26fe3 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:33:47 -0500 Subject: [PATCH 10/34] fix(attachments): show friendly prompt instead of misleading warnings in mapping-file mode --- .../ViewAttachmentFiles.tsx | 40 ++++++++++++++----- .../js_src/lib/localization/attachments.ts | 6 +++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx index a16214b2d93..647a04a71d6 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/ViewAttachmentFiles.tsx @@ -56,13 +56,22 @@ const resolveAttachmentDatasetData = ( typeof status === 'object' && (status.type === 'cancelled' || status.type === 'skipped'); - const statusText = f.maybe(status, resolveAttachmentStatus) ?? ''; + const statusText = + isMappingMode && + status !== undefined && + typeof status === 'object' && + 'reason' in status && + status.reason === 'fileMissing' + ? attachmentsText.awaitingFile() + : f.maybe(status, resolveAttachmentStatus) ?? ''; const baseData = { selectedFileName: [ uploadFile.file.name,
- {uploadFile.file instanceof File ? '' : dialogIcons.warning} + {uploadFile.file instanceof File || isMappingMode + ? '' + : dialogIcons.warning} {uploadFile.file.name}
, ], @@ -178,16 +187,27 @@ export function ViewAttachmentFiles({ })}
- {uploadableFiles.some( - ({ uploadFile: { file } }) => !(file instanceof File) - ) && ( - <> - {dialogIcons.warning} - {attachmentsText.pleaseReselectAllFiles()} - - )} + {!isMappingMode && + uploadableFiles.some( + ({ uploadFile: { file } }) => !(file instanceof File) + ) && ( + <> + {dialogIcons.warning} + {attachmentsText.pleaseReselectAllFiles()} + + )}
+ {isMappingMode && + uploadableFiles.some( + (f) => + f.status?.type === 'cancelled' && + f.status.reason === 'fileMissing' + ) && ( +
+ {attachmentsText.mappingAwaitingFiles()} +
+ )} `bg-[color:var(--background)] p-2 print:p-1 ${ diff --git a/specifyweb/frontend/js_src/lib/localization/attachments.ts b/specifyweb/frontend/js_src/lib/localization/attachments.ts index 33606a7f9f8..cacc15b3f5d 100644 --- a/specifyweb/frontend/js_src/lib/localization/attachments.ts +++ b/specifyweb/frontend/js_src/lib/localization/attachments.ts @@ -964,6 +964,12 @@ export const attachmentsText = createDictionary({ 'hr-hr': 'Datoteka nedostaje', nb: 'Fil mangler', }, + awaitingFile: { + 'en-us': 'Awaiting file', + }, + mappingAwaitingFiles: { + 'en-us': 'Please select your files, or drag and drop them, to begin.', + }, notInMappingFile: { 'en-us': 'Not in Mapping File', 'de-ch': 'Nicht in der Zuordnungsdatei', From 1a572407dcbaf398fdc1a9b903429e09b5de0ca1 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:40:59 -0500 Subject: [PATCH 11/34] feat(attachments): add icon to matching dialog --- .../lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx index 384dac482bf..7cd81aa70af 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { attachmentsText } from '../../localization/attachments'; import { commonText } from '../../localization/common'; import type { RA } from '../../utils/types'; +import { icons } from '../Atoms/Icons'; import { Button } from '../Atoms/Button'; import { Submit } from '../Atoms/Submit'; import { Dialog } from '../Molecules/Dialog'; @@ -49,6 +50,7 @@ export function MatchingModeDialog({ return ( {commonText.close()} From 29ad01bf9441e5736df442596159da8d551df234 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:22:16 -0500 Subject: [PATCH 12/34] fix(attachments): pass oldRows to seed effect and add isMappingMode to deps --- .../AttachmentsBulkImport/Import.tsx | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx index 57cf24e6291..db690a9e75b 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx @@ -212,6 +212,14 @@ function AttachmentsImport({ } }, [applyFileNames, commitFileChange]); + // Single source of truth: dataset-level matchingmode (DB field) or uploadplan-level + const isMappingMode = React.useMemo( + () => + (eagerDataSet.matchingmode ?? eagerDataSet.uploadplan.matchingMode) === + 'mappingFile', + [eagerDataSet.matchingmode, eagerDataSet.uploadplan.matchingMode] + ); + // In mapping mode, seed the table with CSV rows so the user sees what to upload React.useEffect(() => { if ( @@ -220,9 +228,9 @@ function AttachmentsImport({ eagerDataSet.uploadplan.mappingFileData.length > 0 && eagerDataSet.rows.length === 0 ) { - commitFileChange(() => + commitFileChange((oldRows) => crossReferenceMappingFiles( - [], + oldRows, eagerDataSet.uploadplan.mappingFileData! ) ); @@ -232,6 +240,7 @@ function AttachmentsImport({ eagerDataSet.uploadplan.mappingFileData, eagerDataSet.rows.length, commitFileChange, + isMappingMode, ]); const currentBaseTable = @@ -240,14 +249,6 @@ function AttachmentsImport({ : staticAttachmentImportPaths[eagerDataSet.uploadplan.staticPathKey] .baseTable; - // Single source of truth: dataset-level matchingmode (DB field) or uploadplan-level - const isMappingMode = React.useMemo( - () => - (eagerDataSet.matchingmode ?? eagerDataSet.uploadplan.matchingMode) === - 'mappingFile', - [eagerDataSet.matchingmode, eagerDataSet.uploadplan.matchingMode] - ); - const anyUploaded = React.useMemo( () => eagerDataSet.rows.some( From acf35f2e2022d680f3af3c106e377e9c22136289 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:43:02 -0500 Subject: [PATCH 13/34] feat(attachments): detect non-placeholder filename collisions in crossReferenceMappingFiles byName map --- .../lib/components/AttachmentsBulkImport/utils.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index 703393cbc9e..1ef7d55ff51 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -289,6 +289,20 @@ export function crossReferenceMappingFiles( f.uploadFile.file instanceof File) ) { byName.set(f.uploadFile.file.name, f); + } else if (!existingIsPlaceholder && !newIsPlaceholder) { + // Keep the first entry but flag it unless already uploaded. + byName.set(f.uploadFile.file.name, { + ...existing, + ...(existing.attachmentId === undefined && + existing.status?.type !== 'success' + ? { + status: { + type: 'cancelled' as const, + reason: 'duplicateInMappingFile' as const, + }, + } + : {}), + }); } } } From 0abf489e58cb295c042988a74be3e8f1f1bb7074 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:44:58 -0500 Subject: [PATCH 14/34] fix(attachments): reset column indices and dedup ref on new CSV load in MappingFileSetup --- .../components/AttachmentsBulkImport/MappingFileSetup.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx index d2dbcbbfee0..dca8ff8f9bc 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx @@ -154,9 +154,13 @@ export function MappingFileSetup({ const isRestored = initialColumns !== undefined && initialData !== undefined; const [hasRestored] = React.useState(isRestored); + const previousMappingRef = React.useRef(undefined); const handleFileSelected = React.useCallback( async (selectedFile: File) => { setError(undefined); + setMatchValueIndex(undefined); + setFileNameIndex(undefined); + previousMappingRef.current = undefined; try { const parsed = await parseMappingCsv(selectedFile); setHeaders(parsed.headers); @@ -178,7 +182,6 @@ export function MappingFileSetup({ ); // Notify parent when columns are selected - const previousMappingRef = React.useRef(undefined); React.useEffect(() => { if ( matchValueIndex === undefined || From de736b3fdcf307fca86699351704ba12b6224965 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:27:29 -0500 Subject: [PATCH 15/34] fix(attachments): prefer File instances in dedup safety net over first-wins --- .../AttachmentsBulkImport/Import.tsx | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx index db690a9e75b..69027ffe54d 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx @@ -11,7 +11,7 @@ import { userText } from '../../localization/user'; import { wbText } from '../../localization/workbench'; import { ajax } from '../../utils/ajax'; import { f } from '../../utils/functools'; -import type { RA, WritableArray } from '../../utils/types'; +import type { RA } from '../../utils/types'; import type { IR } from '../../utils/types'; import { removeKey, sortFunction } from '../../utils/utils'; import { Container } from '../Atoms'; @@ -269,15 +269,19 @@ function AttachmentsImport({ : matchSelectedFiles(eagerDataSet.rows, filesList); // Safety net: guarantee no duplicate filenames in the final list - const seen = new Set(); - const deduped = (resolvedFiles as WritableArray).filter( - (f) => { - const name = f.uploadFile.file.name; - if (seen.has(name)) return false; - seen.add(name); - return true; + const bestByName = new Map(); + for (const f of resolvedFiles) { + const name = f.uploadFile.file.name; + const existing = bestByName.get(name); + if ( + existing === undefined || + (!(existing.uploadFile.file instanceof File) && + f.uploadFile.file instanceof File) + ) { + bestByName.set(name, f); } - ); + } + const deduped = [...bestByName.values()]; deduped.sort(sortFunction((file) => file.uploadFile.file.name)); commitChange((oldState) => ({ From 3a4a94146d32fab0ed1d0f7eb73859511764236b Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:29:51 -0500 Subject: [PATCH 16/34] fix(attachments): preserve success/attached status in notInMappingFile loop --- .../lib/components/AttachmentsBulkImport/utils.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index 1ef7d55ff51..da24e537d82 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -349,12 +349,18 @@ export function crossReferenceMappingFiles( // Flag any uploaded files not referenced by the CSV for (const [name, file] of byName) { if (!matchedFileNames.has(name)) { + const alreadyAttached = + file.attachmentId !== undefined || file.status?.type === 'success'; result.push({ ...file, - status: { - type: 'cancelled' as const, - reason: 'notInMappingFile' as const, - }, + ...(alreadyAttached + ? {} + : { + status: { + type: 'cancelled' as const, + reason: 'notInMappingFile' as const, + }, + }), }); } } From 7f72e9d30ad28c403eee38f2b819e7d26ce66c80 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:31:05 -0500 Subject: [PATCH 17/34] fix(attachments): use mutable array for crossReferenceMappingFiles result accumulator --- .../js_src/lib/components/AttachmentsBulkImport/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index da24e537d82..771413e96a8 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -307,7 +307,7 @@ export function crossReferenceMappingFiles( } } - const result: RA = []; + const result: PartialUploadableFileSpec[] = []; const matchedFileNames = new Set(); // For each CSV row, emit the best available file or a placeholder From 4eff8426e046ab8bb025811fa1f6fd5d996afa0a Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:32:08 -0500 Subject: [PATCH 18/34] fix(attachments): add mappingFileName to placeholder rows in crossReferenceMappingFiles --- .../js_src/lib/components/AttachmentsBulkImport/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts index 771413e96a8..233421b2afb 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/utils.ts @@ -337,6 +337,7 @@ export function crossReferenceMappingFiles( file: { name: row.fileName, size: 0, type: '' }, parsedName: row.matchValue, mappingMatchValue: row.matchValue, + mappingFileName: row.fileName, }, status: { type: 'cancelled' as const, From f94cd22e3bb40ee052966ff7f4371380bb5faa99 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:40:25 -0500 Subject: [PATCH 19/34] fix(backend): validate matchingmode values and add model-level choices constraint --- specifyweb/backend/attachment_gw/dataset_views.py | 15 +++++++++++++-- specifyweb/backend/attachment_gw/models.py | 9 ++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/specifyweb/backend/attachment_gw/dataset_views.py b/specifyweb/backend/attachment_gw/dataset_views.py index aa055ec22b8..1bb77d93f32 100644 --- a/specifyweb/backend/attachment_gw/dataset_views.py +++ b/specifyweb/backend/attachment_gw/dataset_views.py @@ -12,6 +12,17 @@ class AttachmentDataSetPT(PermissionTarget): upload = PermissionTargetAction() rollback = PermissionTargetAction() +VALID_MATCHING_MODES = {'filename', 'mappingFile'} + +def _get_validated_matchingmode(data, existing=None): + """Return a validated matchingmode value or raise ValueError.""" + if 'matchingmode' not in data: + return existing + value = data['matchingmode'] + if value is not None and value not in VALID_MATCHING_MODES: + raise ValueError(f"Invalid matchingmode: {value!r}") + return value + def datasets_view(request): if request.method == 'GET': return http.JsonResponse(Spattachmentdataset.get_meta_fields(request), safe=False) @@ -28,7 +39,7 @@ def datasets_view(request): createdbyagent=request.specify_user_agent, modifiedbyagent=request.specify_user_agent, uploaderstatus="main", - matchingmode=data.get('matchingmode', None), + matchingmode=_get_validated_matchingmode(data), # A bit more flexible than workbench. Handles creating datasets with an uploadplan from the start. uploadplan=json.dumps(data['uploadplan']) if 'uploadplan' in data else None ) @@ -50,7 +61,7 @@ def dataset_view(request, ds: Spattachmentdataset): ds.name = attrs.get('name', ds.name) ds.remarks = attrs.get('remarks', ds.remarks) ds.data = attrs.get('rows', ds.data) - ds.matchingmode = attrs.get('matchingmode', ds.matchingmode) + ds.matchingmode = _get_validated_matchingmode(attrs, ds.matchingmode) ds.uploadplan = json.dumps(attrs['uploadplan'] if 'uploadplan' in attrs else ds.uploadplan) # Never preserve uploaderstatus. Making it required for all requests. old_status = ds.uploaderstatus diff --git a/specifyweb/backend/attachment_gw/models.py b/specifyweb/backend/attachment_gw/models.py index 81aa3fa0b40..d934c034348 100644 --- a/specifyweb/backend/attachment_gw/models.py +++ b/specifyweb/backend/attachment_gw/models.py @@ -8,7 +8,14 @@ class Spattachmentdataset(Dataset): id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') - matchingmode = models.CharField(max_length=32, null=True, default=None) + MATCHING_MODE_CHOICES = [ + ('filename', 'Filename'), + ('mappingFile', 'Mapping File'), + ] + + matchingmode = models.CharField( + max_length=32, null=True, default=None, choices=MATCHING_MODE_CHOICES + ) object_response_fields = [ *Dataset.object_response_fields, From b4ae1bf4344cc396a5c90a50ae2bcd8c72473f1e Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:41:49 -0500 Subject: [PATCH 20/34] fix(attachments): clear headers and rows on CSV parse error in MappingFileSetup --- .../lib/components/AttachmentsBulkImport/MappingFileSetup.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx index dca8ff8f9bc..a53db7418d0 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx @@ -173,6 +173,8 @@ export function MappingFileSetup({ if (detected.fileNameIndex !== undefined) setFileNameIndex(detected.fileNameIndex); } catch (err) { + setHeaders(undefined); + setRows(undefined); setError( err instanceof Error ? err.message : 'Failed to parse CSV file' ); From a0383b2a590b177f2c04b5193ecaedda2e63a753 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:47:25 -0500 Subject: [PATCH 21/34] fix(attachments): anchor fileNamePatterns to prevent substring matches --- .../AttachmentsBulkImport/MappingFileSetup.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx index a53db7418d0..cb3ad733d1b 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MappingFileSetup.tsx @@ -96,11 +96,11 @@ function autoDetectColumns(headers: RA): { const fieldMatchers = buildFieldMatchers(); const fileNamePatterns = [ - /attachment\s*name/i, - /file\s*name/i, - /filename/i, - /attachment/i, - /file/i, + /^attachment\s*name$/i, + /^file\s*name$/i, + /^filename$/i, + /^attachment$/i, + /^file$/i, ]; let matchValueIndex: number | undefined; From feb6e2578577a46c9d3be19feb5afbc633ed1c77 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:06:31 -0500 Subject: [PATCH 22/34] refactor(attachments): treat null matchingmode as filename mode instead of storing 'filename' --- .../backend/attachment_gw/dataset_views.py | 2 +- specifyweb/backend/attachment_gw/models.py | 1 - .../components/AttachmentsBulkImport/Import.tsx | 6 +++--- .../AttachmentsBulkImport/MatchingModeDialog.tsx | 16 ++++++++-------- .../components/AttachmentsBulkImport/types.ts | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/specifyweb/backend/attachment_gw/dataset_views.py b/specifyweb/backend/attachment_gw/dataset_views.py index 1bb77d93f32..ab69d4ef71d 100644 --- a/specifyweb/backend/attachment_gw/dataset_views.py +++ b/specifyweb/backend/attachment_gw/dataset_views.py @@ -12,7 +12,7 @@ class AttachmentDataSetPT(PermissionTarget): upload = PermissionTargetAction() rollback = PermissionTargetAction() -VALID_MATCHING_MODES = {'filename', 'mappingFile'} +VALID_MATCHING_MODES = {'mappingFile'} def _get_validated_matchingmode(data, existing=None): """Return a validated matchingmode value or raise ValueError.""" diff --git a/specifyweb/backend/attachment_gw/models.py b/specifyweb/backend/attachment_gw/models.py index d934c034348..4058e4b88b3 100644 --- a/specifyweb/backend/attachment_gw/models.py +++ b/specifyweb/backend/attachment_gw/models.py @@ -9,7 +9,6 @@ class Spattachmentdataset(Dataset): id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') MATCHING_MODE_CHOICES = [ - ('filename', 'Filename'), ('mappingFile', 'Mapping File'), ] diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx index 69027ffe54d..fd3e6109a6b 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/Import.tsx @@ -58,7 +58,7 @@ export type AttachmentUploadSpec = { }; export type PartialAttachmentUploadSpec = { readonly fieldFormatter?: UiFormatter; - readonly matchingMode?: MatchingMode; + readonly matchingMode?: MatchingMode | null; readonly mappingFileColumns?: MappingFileColumns; readonly mappingFileData?: RA; } & (AttachmentUploadSpec | { readonly staticPathKey: undefined }); @@ -578,10 +578,10 @@ function AttachmentsImport({ // Default to filename mode if user closes without choosing commitChange((oldState) => ({ ...oldState, - matchingmode: 'filename' as MatchingMode, + matchingmode: null, uploadplan: { ...oldState.uploadplan, - matchingMode: 'filename' as MatchingMode, + matchingMode: null, }, })); }} diff --git a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx index 7cd81aa70af..952bce84dc1 100644 --- a/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx +++ b/specifyweb/frontend/js_src/lib/components/AttachmentsBulkImport/MatchingModeDialog.tsx @@ -22,17 +22,17 @@ export function MatchingModeDialog({ initialData, }: { readonly onContinue: ( - mode: MatchingMode, + mode: MatchingMode | null, columns?: MappingFileColumns, data?: RA ) => void; readonly onClose: () => void; - readonly initialMode?: MatchingMode; + readonly initialMode?: MatchingMode | null; readonly initialColumns?: MappingFileColumns; readonly initialData?: RA; }): JSX.Element { - const [mode, setMode] = React.useState( - initialMode ?? 'filename' + const [mode, setMode] = React.useState( + initialMode ?? null ); const [mappingColumns, setMappingColumns] = React.useState< MappingFileColumns | undefined @@ -42,7 +42,7 @@ export function MatchingModeDialog({ >(initialData); const canContinue = - mode === 'filename' || + (mode === null || mode === undefined) || (mode === 'mappingFile' && mappingColumns !== undefined && mappingData !== undefined && @@ -72,17 +72,17 @@ export function MatchingModeDialog({