From c0d7cba2037ce2d13b7f653ce7cdcee7995af1d1 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Mon, 13 May 2024 13:13:58 -0400 Subject: [PATCH 01/59] Display cat number range for disctinct queries Fixes #2000 --- .../lib/components/QueryBuilder/ResultsTable.tsx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index 2dd906a7107..c50a0643a3e 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -181,12 +181,19 @@ function Row({ rel="noreferrer" /> ) : ( - toggleIsListOfRecordsOpen(true)} + // /> + toggleIsListOfRecordsOpen(true)} - /> + > + {`${splitRecords[0]} - ${splitRecords.at(-1)}`} + )} {isListOfRecordsOpen && splitRecords !== undefined ? ( Date: Tue, 14 May 2024 07:53:34 -0700 Subject: [PATCH 02/59] Comment out --- .../components/QueryBuilder/ResultsTable.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index c50a0643a3e..bd2e27a4d89 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -181,19 +181,19 @@ function Row({ rel="noreferrer" /> ) : ( - // toggleIsListOfRecordsOpen(true)} - // /> - toggleIsListOfRecordsOpen(true)} - > - {`${splitRecords[0]} - ${splitRecords.at(-1)}`} - + /> + // toggleIsListOfRecordsOpen(true)} + // > + // {`${splitRecords[0]} - ${splitRecords.at(-1)}`} + // )} {isListOfRecordsOpen && splitRecords !== undefined ? ( Date: Tue, 14 May 2024 13:56:01 -0700 Subject: [PATCH 03/59] test series on front end --- .../lib/components/QueryBuilder/Results.tsx | 2 + .../components/QueryBuilder/ResultsTable.tsx | 64 ++++++++++++++++--- .../QueryBuilder/ResultsWrapper.tsx | 3 + .../lib/components/QueryBuilder/Toolbar.tsx | 13 ++++ .../lib/components/QueryBuilder/Wrapped.tsx | 7 +- .../frontend/js_src/lib/localization/query.ts | 3 + 6 files changed, 83 insertions(+), 9 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx index 7fbe27fcc8b..03c801cf484 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx @@ -69,6 +69,7 @@ export type QueryResultsProps = { readonly resultsRef?: React.MutableRefObject< RA | undefined >; + readonly isSeries?: boolean; }; export function QueryResults(props: QueryResultsProps): JSX.Element { @@ -365,6 +366,7 @@ export function QueryResults(props: QueryResultsProps): JSX.Element { lastSelectedRow.current = rowIndex; }} + isSeries={props.isSeries} /> ) : undefined} {isFetching || (!showResults && Array.isArray(results)) ? ( diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index bd2e27a4d89..c03959e2ed3 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -26,6 +26,7 @@ export function QueryResultsTable({ results, selectedRows, onSelected: handleSelected, + isSeries, }: { readonly table: SpecifyTable; readonly fieldSpecs: RA; @@ -36,6 +37,7 @@ export function QueryResultsTable({ isSelected: boolean, isShiftClick: boolean ) => void; + readonly isSeries: boolean; }): JSX.Element { const recordFormatter = React.useMemo( () => getAuditRecordFormatter(fieldSpecs), @@ -61,6 +63,7 @@ export function QueryResultsTable({ onSelected={(isSelected, isShiftClick): void => handleSelected(index, isSelected, isShiftClick) } + isSeries={isSeries} /> ))} @@ -76,6 +79,7 @@ function Row({ isSelected, isLast, onSelected: handleSelected, + isSeries, }: { readonly table: SpecifyTable; readonly fieldSpecs: RA; @@ -87,6 +91,7 @@ function Row({ readonly isSelected: boolean; readonly isLast: boolean; readonly onSelected?: (isSelected: boolean, isShiftClick: boolean) => void; + readonly isSeries: boolean; }): JSX.Element { // REFACTOR: replace this with getResourceViewUrl() const [resource] = useLiveState< @@ -122,6 +127,32 @@ function Row({ [result] ); + const resultB = result.slice(1); + const ranges: (number | null | string)[][] = []; + if (splitRecords) { + let currentRange: number[] = [splitRecords[0]]; + + for (let i = 1; i < splitRecords.length; i++) { + if (splitRecords[i] === splitRecords[i - 1] + 1) { + currentRange.push(splitRecords[i]); + } else { + ranges.push([...currentRange]); + currentRange = [splitRecords[i]]; + } + } + + ranges.push([...currentRange]); + } + + const resultVariables = ranges.map((range, index) => { + const resultVarName = `result${index + 1}`; + return { [resultVarName]: [...range, ...resultB] }; + }); + + const results2 = Object.assign({}, ...resultVariables); + + console.log(results2); + const [isListOfRecordsOpen, toggleIsListOfRecordsOpen] = React.useState(false); @@ -180,6 +211,14 @@ function Row({ href={viewUrl} rel="noreferrer" /> + ) : isSeries ? ( + toggleIsListOfRecordsOpen(true)} + > + {splitRecords.length} + ) : ( toggleIsListOfRecordsOpen(true)} /> - // toggleIsListOfRecordsOpen(true)} - // > - // {`${splitRecords[0]} - ${splitRecords.at(-1)}`} - // )} {isListOfRecordsOpen && splitRecords !== undefined ? ( )} - {result + {/* {result + .filter((_, index) => index !== queryIdField) + .map((value, index) => + fieldSpecs[index].isPhantom ? undefined : ( + + ) + )} */} + {Object.entries(results2) .filter((_, index) => index !== queryIdField) .map((value, index) => fieldSpecs[index].isPhantom ? undefined : ( diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsWrapper.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsWrapper.tsx index 04efb26a676..6917c6907a2 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsWrapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsWrapper.tsx @@ -33,12 +33,14 @@ export function QueryResultsWrapper({ extraButtons, onSelected: handleSelected, onReRun: handleReRun, + isSeries, ...props }: ResultsProps & { readonly createRecordSet: JSX.Element | undefined; readonly extraButtons: JSX.Element | undefined; readonly onSelected?: (selected: RA) => void; readonly onReRun: () => void; + readonly isSeries: boolean; }): JSX.Element | null { const newProps = useQueryResultsWrapper(props); @@ -55,6 +57,7 @@ export function QueryResultsWrapper({ extraButtons={extraButtons} onReRun={handleReRun} onSelected={handleSelected} + isSeries={isSeries} /> diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx index 4dc73dbf72c..5bd31e0ddcb 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx @@ -13,16 +13,20 @@ export function QueryToolbar({ showHiddenFields, tableName, isDistinct, + isSeries, onToggleHidden: handleToggleHidden, onToggleDistinct: handleToggleDistinct, + onToggleSeries: handleToggleSeries, onRunCountOnly: handleRunCountOnly, onSubmitClick: handleSubmitClick, }: { readonly showHiddenFields: boolean; readonly tableName: keyof Tables; readonly isDistinct: boolean; + readonly isSeries: boolean; readonly onToggleHidden: (value: boolean) => void; readonly onToggleDistinct: () => void; + readonly onToggleSeries: () => void; readonly onRunCountOnly: () => void; readonly onSubmitClick: () => void; }): JSX.Element { @@ -38,6 +42,15 @@ export function QueryToolbar({ {hasPermission('/querybuilder/query', 'execute') && ( <> + {tableName === 'CollectionObject' && ( + + + {queryText.series()} + + )} {/* * Query Distinct for trees is disabled because of * https://github.com/specify/specify7/pull/1019#issuecomment-973525594 diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 7cfe981d1f8..20419459c18 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -128,6 +128,8 @@ function Wrapped({ const [saveRequired, setSaveRequired] = React.useState(false); + const [isSeries, setIsSeries] = React.useState(false); + React.useEffect(() => { const initialState = buildInitialState(); dispatch({ @@ -158,7 +160,7 @@ function Wrapped({ fields: unParseQueryFields(state.baseTableName, state.fields), isDistinct: query.selectDistinct, }); - }, [state, query.selectDistinct]); + }, [state, query.selectDistinct, isSeries]); /** * If tried to save a query, enforce the field length limit for the @@ -556,6 +558,7 @@ function Wrapped({ /> runQuery('count')} @@ -570,6 +573,7 @@ function Wrapped({ selectDistinct: !(query.selectDistinct ?? false), }) } + onToggleSeries={(): void => setIsSeries(!isSeries)} onToggleHidden={setShowHiddenFields} /> @@ -621,6 +625,7 @@ function Wrapped({ }); runQuery('regular', fields); }} + isSeries={isSeries} /> )} diff --git a/specifyweb/frontend/js_src/lib/localization/query.ts b/specifyweb/frontend/js_src/lib/localization/query.ts index 4f0b08c2775..5a88ebe9c1b 100644 --- a/specifyweb/frontend/js_src/lib/localization/query.ts +++ b/specifyweb/frontend/js_src/lib/localization/query.ts @@ -300,6 +300,9 @@ export const queryText = createDictionary({ 'uk-ua': 'Виразний', 'de-ch': 'Unterscheidbar', }, + series: { + 'en-us': 'Series', + }, createCsv: { 'en-us': 'Create CSV', 'ru-ru': 'Создать CSV-файл', From ee0813a25fbd795f0e0e68b77ca7f852b1d24aec Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Wed, 15 May 2024 10:19:24 -0700 Subject: [PATCH 04/59] Test grouping results by following ids --- .../components/QueryBuilder/ResultsTable.tsx | 121 ++++++++++-------- 1 file changed, 67 insertions(+), 54 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index c03959e2ed3..34e21179321 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -48,9 +48,16 @@ export function QueryResultsTable({ 'appearance', 'showLineNumber' ); + + const splitRecordsList = results.map((result) => + typeof result[0] === 'string' && result[0].includes(',') + ? result[0].split(',').map(Number) + : undefined + ); + return ( <> - {results.map((result, index, { length }) => ( + {/* {results.map((result, index, { length }) => ( - ))} + ))} */} + {results.map((result, index, { length }) => { + const splitRecords = splitRecordsList[index]; + + const resultFields = result.slice(1); + const ranges: (number | null | string)[][] = []; + if (splitRecords) { + let currentRange: number[] = [splitRecords[0]]; + + for (let i = 1; i < splitRecords.length; i++) { + if (splitRecords[i] === splitRecords[i - 1] + 1) { + currentRange.push(splitRecords[i]); + } else { + ranges.push([...currentRange]); + currentRange = [splitRecords[i]]; + } + } + + ranges.push([...currentRange]); + } + + const resultVariables = ranges.map((range, index) => { + const resultVarName = `result${index + 1}`; + const modifiedRange = range.join(','); + return { [resultVarName]: [modifiedRange, ...resultFields] }; + }); + + const resultsSplitted = Object.assign({}, ...resultVariables); + + return Object.keys(resultsSplitted).map((key) => ( + + handleSelected(index, isSelected, isShiftClick) + } + isSeries={isSeries} + /> + )); + })} ); } @@ -127,32 +181,6 @@ function Row({ [result] ); - const resultB = result.slice(1); - const ranges: (number | null | string)[][] = []; - if (splitRecords) { - let currentRange: number[] = [splitRecords[0]]; - - for (let i = 1; i < splitRecords.length; i++) { - if (splitRecords[i] === splitRecords[i - 1] + 1) { - currentRange.push(splitRecords[i]); - } else { - ranges.push([...currentRange]); - currentRange = [splitRecords[i]]; - } - } - - ranges.push([...currentRange]); - } - - const resultVariables = ranges.map((range, index) => { - const resultVarName = `result${index + 1}`; - return { [resultVarName]: [...range, ...resultB] }; - }); - - const results2 = Object.assign({}, ...resultVariables); - - console.log(results2); - const [isListOfRecordsOpen, toggleIsListOfRecordsOpen] = React.useState(false); @@ -211,21 +239,22 @@ function Row({ href={viewUrl} rel="noreferrer" /> - ) : isSeries ? ( + ) : ( + // toggleIsListOfRecordsOpen(true)} + // /> toggleIsListOfRecordsOpen(true)} > - {splitRecords.length} + {`${splitRecords[0]} - ${splitRecords.at(-1)} | ${ + splitRecords.length + }`} - ) : ( - toggleIsListOfRecordsOpen(true)} - /> )} {isListOfRecordsOpen && splitRecords !== undefined ? ( )} - {/* {result - .filter((_, index) => index !== queryIdField) - .map((value, index) => - fieldSpecs[index].isPhantom ? undefined : ( - - ) - )} */} - {Object.entries(results2) + {result .filter((_, index) => index !== queryIdField) .map((value, index) => fieldSpecs[index].isPhantom ? undefined : ( From 42ed6c5ed6d912bbf9b01318be5f586af2f35c76 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Wed, 15 May 2024 11:04:32 -0700 Subject: [PATCH 05/59] Test condition --- .../components/QueryBuilder/ResultsTable.tsx | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index 34e21179321..e189e61b77b 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -55,24 +55,8 @@ export function QueryResultsTable({ : undefined ); - return ( + return isSeries ? ( <> - {/* {results.map((result, index, { length }) => ( - - handleSelected(index, isSelected, isShiftClick) - } - isSeries={isSeries} - /> - ))} */} {results.map((result, index, { length }) => { const splitRecords = splitRecordsList[index]; @@ -121,6 +105,25 @@ export function QueryResultsTable({ )); })} + ) : ( + <> + {results.map((result, index, { length }) => ( + + handleSelected(index, isSelected, isShiftClick) + } + isSeries={isSeries} + /> + ))} + ); } @@ -239,13 +242,7 @@ function Row({ href={viewUrl} rel="noreferrer" /> - ) : ( - // toggleIsListOfRecordsOpen(true)} - // /> + ) : isSeries ? ( + ) : ( + toggleIsListOfRecordsOpen(true)} + /> )} {isListOfRecordsOpen && splitRecords !== undefined ? ( Date: Wed, 15 May 2024 11:20:07 -0700 Subject: [PATCH 06/59] Try reduce --- .../components/QueryBuilder/ResultsTable.tsx | 65 +++++++++++++------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index e189e61b77b..6fc152cb7bd 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -61,31 +61,56 @@ export function QueryResultsTable({ const splitRecords = splitRecordsList[index]; const resultFields = result.slice(1); - const ranges: (number | null | string)[][] = []; - if (splitRecords) { - let currentRange: number[] = [splitRecords[0]]; + // const ranges: (number | null | string)[][] = []; + // if (splitRecords) { + // let currentRange: number[] = [splitRecords[0]]; - for (let i = 1; i < splitRecords.length; i++) { - if (splitRecords[i] === splitRecords[i - 1] + 1) { - currentRange.push(splitRecords[i]); - } else { - ranges.push([...currentRange]); - currentRange = [splitRecords[i]]; - } - } + // for (let i = 1; i < splitRecords.length; i++) { + // if (splitRecords[i] === splitRecords[i - 1] + 1) { + // currentRange.push(splitRecords[i]); + // } else { + // ranges.push([...currentRange]); + // currentRange = [splitRecords[i]]; + // } + // } + + // ranges.push([...currentRange]); + // } - ranges.push([...currentRange]); - } + // const resultVariables = ranges.map((range, index) => { + // const resultVarName = `result${index + 1}`; + // const modifiedRange = range.join(','); + // return { [resultVarName]: [modifiedRange, ...resultFields] }; + // }); - const resultVariables = ranges.map((range, index) => { + // const resultsSplitted = Object.assign({}, ...resultVariables); + + const ranges1 = splitRecords + ? splitRecords.reduce( + (accumulator: number[][], currentValue, index) => { + if ( + index === 0 || + currentValue !== splitRecords[index - 1] + 1 + ) { + accumulator.push([]); + } + accumulator[accumulator.length - 1].push(currentValue); + return accumulator; + }, + [] + ) + : []; + + const resultsSplitted1 = ranges1.reduce((accumulator, range, index) => { const resultVarName = `result${index + 1}`; const modifiedRange = range.join(','); - return { [resultVarName]: [modifiedRange, ...resultFields] }; - }); - - const resultsSplitted = Object.assign({}, ...resultVariables); + return { + ...accumulator, + [resultVarName]: [modifiedRange, ...resultFields], + }; + }, {}); - return Object.keys(resultsSplitted).map((key) => ( + return Object.keys(resultsSplitted1).map((key) => ( handleSelected(index, isSelected, isShiftClick) From 36c6c6d61a4d7e0a1c62a2b18fbb65b63bbd8c8c Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 08:51:28 -0700 Subject: [PATCH 07/59] comment --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 2aaa201dcab..c2b3440a5b6 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -539,7 +539,7 @@ def execute(session, collection, user, tableid, distinct, count_only, field_spec query = query.order_by(*order_by_exprs).offset(offset) if limit: query = query.limit(limit) - +# query.groupBy('catalogNumber') return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, From 26cedcc2b16b02a28035c152b80ed0d5685054b8 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 10:08:43 -0700 Subject: [PATCH 08/59] Reset --- .../lib/components/QueryBuilder/Results.tsx | 2 - .../components/QueryBuilder/ResultsTable.tsx | 99 +------------------ .../QueryBuilder/ResultsWrapper.tsx | 3 - .../lib/components/QueryBuilder/Toolbar.tsx | 9 +- .../lib/components/QueryBuilder/Wrapped.tsx | 7 +- 5 files changed, 3 insertions(+), 117 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx index 03c801cf484..7fbe27fcc8b 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Results.tsx @@ -69,7 +69,6 @@ export type QueryResultsProps = { readonly resultsRef?: React.MutableRefObject< RA | undefined >; - readonly isSeries?: boolean; }; export function QueryResults(props: QueryResultsProps): JSX.Element { @@ -366,7 +365,6 @@ export function QueryResults(props: QueryResultsProps): JSX.Element { lastSelectedRow.current = rowIndex; }} - isSeries={props.isSeries} /> ) : undefined} {isFetching || (!showResults && Array.isArray(results)) ? ( diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx index 6fc152cb7bd..2dd906a7107 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/ResultsTable.tsx @@ -26,7 +26,6 @@ export function QueryResultsTable({ results, selectedRows, onSelected: handleSelected, - isSeries, }: { readonly table: SpecifyTable; readonly fieldSpecs: RA; @@ -37,7 +36,6 @@ export function QueryResultsTable({ isSelected: boolean, isShiftClick: boolean ) => void; - readonly isSeries: boolean; }): JSX.Element { const recordFormatter = React.useMemo( () => getAuditRecordFormatter(fieldSpecs), @@ -48,89 +46,7 @@ export function QueryResultsTable({ 'appearance', 'showLineNumber' ); - - const splitRecordsList = results.map((result) => - typeof result[0] === 'string' && result[0].includes(',') - ? result[0].split(',').map(Number) - : undefined - ); - - return isSeries ? ( - <> - {results.map((result, index, { length }) => { - const splitRecords = splitRecordsList[index]; - - const resultFields = result.slice(1); - // const ranges: (number | null | string)[][] = []; - // if (splitRecords) { - // let currentRange: number[] = [splitRecords[0]]; - - // for (let i = 1; i < splitRecords.length; i++) { - // if (splitRecords[i] === splitRecords[i - 1] + 1) { - // currentRange.push(splitRecords[i]); - // } else { - // ranges.push([...currentRange]); - // currentRange = [splitRecords[i]]; - // } - // } - - // ranges.push([...currentRange]); - // } - - // const resultVariables = ranges.map((range, index) => { - // const resultVarName = `result${index + 1}`; - // const modifiedRange = range.join(','); - // return { [resultVarName]: [modifiedRange, ...resultFields] }; - // }); - - // const resultsSplitted = Object.assign({}, ...resultVariables); - - const ranges1 = splitRecords - ? splitRecords.reduce( - (accumulator: number[][], currentValue, index) => { - if ( - index === 0 || - currentValue !== splitRecords[index - 1] + 1 - ) { - accumulator.push([]); - } - accumulator[accumulator.length - 1].push(currentValue); - return accumulator; - }, - [] - ) - : []; - - const resultsSplitted1 = ranges1.reduce((accumulator, range, index) => { - const resultVarName = `result${index + 1}`; - const modifiedRange = range.join(','); - return { - ...accumulator, - [resultVarName]: [modifiedRange, ...resultFields], - }; - }, {}); - - return Object.keys(resultsSplitted1).map((key) => ( - - handleSelected(index, isSelected, isShiftClick) - } - isSeries={isSeries} - /> - )); - })} - - ) : ( + return ( <> {results.map((result, index, { length }) => ( handleSelected(index, isSelected, isShiftClick) } - isSeries={isSeries} /> ))} @@ -161,7 +76,6 @@ function Row({ isSelected, isLast, onSelected: handleSelected, - isSeries, }: { readonly table: SpecifyTable; readonly fieldSpecs: RA; @@ -173,7 +87,6 @@ function Row({ readonly isSelected: boolean; readonly isLast: boolean; readonly onSelected?: (isSelected: boolean, isShiftClick: boolean) => void; - readonly isSeries: boolean; }): JSX.Element { // REFACTOR: replace this with getResourceViewUrl() const [resource] = useLiveState< @@ -267,16 +180,6 @@ function Row({ href={viewUrl} rel="noreferrer" /> - ) : isSeries ? ( - toggleIsListOfRecordsOpen(true)} - > - {`${splitRecords[0]} - ${splitRecords.at(-1)} | ${ - splitRecords.length - }`} - ) : ( ) => void; readonly onReRun: () => void; - readonly isSeries: boolean; }): JSX.Element | null { const newProps = useQueryResultsWrapper(props); @@ -57,7 +55,6 @@ export function QueryResultsWrapper({ extraButtons={extraButtons} onReRun={handleReRun} onSelected={handleSelected} - isSeries={isSeries} /> diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx index 5bd31e0ddcb..7acdc33de9a 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx @@ -13,20 +13,16 @@ export function QueryToolbar({ showHiddenFields, tableName, isDistinct, - isSeries, onToggleHidden: handleToggleHidden, onToggleDistinct: handleToggleDistinct, - onToggleSeries: handleToggleSeries, onRunCountOnly: handleRunCountOnly, onSubmitClick: handleSubmitClick, }: { readonly showHiddenFields: boolean; readonly tableName: keyof Tables; readonly isDistinct: boolean; - readonly isSeries: boolean; readonly onToggleHidden: (value: boolean) => void; readonly onToggleDistinct: () => void; - readonly onToggleSeries: () => void; readonly onRunCountOnly: () => void; readonly onSubmitClick: () => void; }): JSX.Element { @@ -44,10 +40,7 @@ export function QueryToolbar({ <> {tableName === 'CollectionObject' && ( - + void} /> {queryText.series()} )} diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 20419459c18..7cfe981d1f8 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -128,8 +128,6 @@ function Wrapped({ const [saveRequired, setSaveRequired] = React.useState(false); - const [isSeries, setIsSeries] = React.useState(false); - React.useEffect(() => { const initialState = buildInitialState(); dispatch({ @@ -160,7 +158,7 @@ function Wrapped({ fields: unParseQueryFields(state.baseTableName, state.fields), isDistinct: query.selectDistinct, }); - }, [state, query.selectDistinct, isSeries]); + }, [state, query.selectDistinct]); /** * If tried to save a query, enforce the field length limit for the @@ -558,7 +556,6 @@ function Wrapped({ /> runQuery('count')} @@ -573,7 +570,6 @@ function Wrapped({ selectDistinct: !(query.selectDistinct ?? false), }) } - onToggleSeries={(): void => setIsSeries(!isSeries)} onToggleHidden={setShowHiddenFields} /> @@ -625,7 +621,6 @@ function Wrapped({ }); runQuery('regular', fields); }} - isSeries={isSeries} /> )} From 36481c32cf629d49fbd779139c4d1ec6931045b0 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 10:22:09 -0700 Subject: [PATCH 09/59] Add series to frontend --- .../components/DataModel/__tests__/helpers.test.ts | 1 + .../frontend/js_src/lib/components/DataModel/types.ts | 1 + .../components/Forms/__tests__/DeleteButton.test.tsx | 1 + .../js_src/lib/components/QueryBuilder/Toolbar.tsx | 9 ++++++++- .../js_src/lib/components/QueryBuilder/Wrapped.tsx | 11 ++++++++++- .../js_src/lib/components/QueryBuilder/index.tsx | 1 + .../js_src/lib/components/QueryComboBox/helpers.ts | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts index 8f7b0e49b9f..12247234d9b 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/helpers.test.ts @@ -122,6 +122,7 @@ describe('serializeResource', () => { remarks: null, searchSynonymy: null, selectDistinct: null, + selectSeries: null, smushed: null, specifyUser: null, sqlStr: null, diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/types.ts b/specifyweb/frontend/js_src/lib/components/DataModel/types.ts index 31aa487613d..f768be3db13 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/types.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/types.ts @@ -5251,6 +5251,7 @@ export type SpQuery = { readonly remarks: string | null; readonly searchSynonymy: boolean | null; readonly selectDistinct: boolean | null; + readonly selectSeries: boolean | null; readonly smushed: boolean | null; readonly sqlStr: string | null; readonly timestampCreated: string; diff --git a/specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx b/specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx index c6191830a03..d43d5d618b8 100644 --- a/specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx +++ b/specifyweb/frontend/js_src/lib/components/Forms/__tests__/DeleteButton.test.tsx @@ -103,6 +103,7 @@ overrideAjax( resource_uri: undefined, searchsynonymy: null, selectdistinct: false, + selectseries: false, smushed: null, specifyuser: '/api/specify/specifyuser/2/', sqlstr: null, diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx index 7acdc33de9a..5bd31e0ddcb 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx @@ -13,16 +13,20 @@ export function QueryToolbar({ showHiddenFields, tableName, isDistinct, + isSeries, onToggleHidden: handleToggleHidden, onToggleDistinct: handleToggleDistinct, + onToggleSeries: handleToggleSeries, onRunCountOnly: handleRunCountOnly, onSubmitClick: handleSubmitClick, }: { readonly showHiddenFields: boolean; readonly tableName: keyof Tables; readonly isDistinct: boolean; + readonly isSeries: boolean; readonly onToggleHidden: (value: boolean) => void; readonly onToggleDistinct: () => void; + readonly onToggleSeries: () => void; readonly onRunCountOnly: () => void; readonly onSubmitClick: () => void; }): JSX.Element { @@ -40,7 +44,10 @@ export function QueryToolbar({ <> {tableName === 'CollectionObject' && ( - void} /> + {queryText.series()} )} diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 7cfe981d1f8..da9b7857506 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -94,6 +94,7 @@ function Wrapped({ readonly onChange?: (props: { readonly fields: RA>; readonly isDistinct: boolean | null; + readonly isSeries: boolean | null; }) => void; }): JSX.Element { const [query, setQuery] = useResource(queryResource); @@ -157,8 +158,9 @@ function Wrapped({ handleChange?.({ fields: unParseQueryFields(state.baseTableName, state.fields), isDistinct: query.selectDistinct, + isSeries: query.selectSeries, }); - }, [state, query.selectDistinct]); + }, [state, query.selectDistinct, query.selectSeries]); /** * If tried to save a query, enforce the field length limit for the @@ -556,6 +558,7 @@ function Wrapped({ /> runQuery('count')} @@ -570,6 +573,12 @@ function Wrapped({ selectDistinct: !(query.selectDistinct ?? false), }) } + onToggleSeries={(): void => + setQuery({ + ...query, + selectSeries: !(query.selectSeries ?? false), + }) + } onToggleHidden={setShowHiddenFields} /> diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx index 37e36803e15..b9ef427a5ef 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/index.tsx @@ -97,6 +97,7 @@ export function createQuery( query.set('contextName', table.name); query.set('contextTableId', table.tableId); query.set('selectDistinct', false); + query.set('selectSeries', false); query.set('countOnly', false); query.set('formatAuditRecIds', false); query.set('specifyUser', userInformation.resource_uri); diff --git a/specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts b/specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts index 0a3651d8e8f..9063cb4aa89 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/QueryComboBox/helpers.ts @@ -35,6 +35,7 @@ export function makeComboBoxQuery({ query.set('contextName', table.name); query.set('contextTableId', table.tableId); query.set('selectDistinct', false); + query.set('selectSeries', false); query.set('countOnly', false); query.set('specifyUser', userInformation.resource_uri); query.set('isFavorite', false); From 5ee0e807115e36cbd57ba81bb8d4f8ce9c1b85fc Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 10:30:48 -0700 Subject: [PATCH 10/59] Only display series for CO with Cat Num in fields --- .../frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx | 4 +++- .../frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx index 5bd31e0ddcb..4372fdc2a54 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx @@ -14,6 +14,7 @@ export function QueryToolbar({ tableName, isDistinct, isSeries, + showSeries, onToggleHidden: handleToggleHidden, onToggleDistinct: handleToggleDistinct, onToggleSeries: handleToggleSeries, @@ -24,6 +25,7 @@ export function QueryToolbar({ readonly tableName: keyof Tables; readonly isDistinct: boolean; readonly isSeries: boolean; + readonly showSeries: boolean; readonly onToggleHidden: (value: boolean) => void; readonly onToggleDistinct: () => void; readonly onToggleSeries: () => void; @@ -42,7 +44,7 @@ export function QueryToolbar({ {hasPermission('/querybuilder/query', 'execute') && ( <> - {tableName === 'CollectionObject' && ( + {showSeries && ( => treeRanksPromise.then(f.true); @@ -298,6 +299,10 @@ function Wrapped({ undefined ); + const showSeries = + table.name === 'CollectionObject' && + state.fields.find((field) => field.mappingPath[0] === 'catalogNumber'); + return treeRanksLoaded ? ( @@ -559,6 +564,7 @@ function Wrapped({ runQuery('count')} From 186a0f67841e0facbd1785a298796cb468aa6d90 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 10:33:34 -0700 Subject: [PATCH 11/59] Fix boolean --- .../frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index d81cfe79b85..652d214260e 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -301,7 +301,7 @@ function Wrapped({ const showSeries = table.name === 'CollectionObject' && - state.fields.find((field) => field.mappingPath[0] === 'catalogNumber'); + state.fields.some((field) => field.mappingPath[0] === 'catalogNumber'); return treeRanksLoaded ? ( From 70c84d702a20e6e94c02104c575afd5962a09569 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 10:37:42 -0700 Subject: [PATCH 12/59] Remove import --- .../frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 652d214260e..bd50313bda1 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -52,7 +52,6 @@ import { getInitialState, reducer } from './reducer'; import type { QueryResultRow } from './Results'; import { QueryResultsWrapper } from './ResultsWrapper'; import { QueryToolbar } from './Toolbar'; -import { Fields } from '../Formatters/Fields'; const fetchTreeRanks = async (): Promise => treeRanksPromise.then(f.true); From eba3b4cd8535195feef01117520858ba48a024fc Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 11:26:56 -0700 Subject: [PATCH 13/59] Start to add series to backend --- specifyweb/stored_queries/execution.py | 11 ++++++++--- specifyweb/stored_queries/views.py | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index c2b3440a5b6..6ee3a7482c0 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -386,6 +386,7 @@ def run_ephemeral_query(collection, user, spquery): offset = spquery.get('offset', 0) recordsetid = spquery.get('recordsetid', None) distinct = spquery['selectdistinct'] + series = spquery['selectseries'] tableid = spquery['contexttableid'] count_only = spquery['countonly'] try: @@ -395,7 +396,7 @@ def run_ephemeral_query(collection, user, spquery): with models.session_context() as session: field_specs = field_specs_from_json(spquery['fields']) - return execute(session, collection, user, tableid, distinct, count_only, + return execute(session, collection, user, tableid, distinct, series, count_only, field_specs, limit, offset, recordsetid, formatauditobjs=format_audits) def augment_field_specs(field_specs, formatauditobjs=False): @@ -526,7 +527,7 @@ def return_loan_preps(collection, user, agent, data): ]) return to_return -def execute(session, collection, user, tableid, distinct, count_only, field_specs, limit, offset, recordsetid=None, formatauditobjs=False): +def execute(session, collection, user, tableid, distinct, series, count_only, field_specs, limit, offset, recordsetid=None, formatauditobjs=False): "Build and execute a query, returning the results as a data structure for json serialization" set_group_concat_max_len(session) @@ -539,7 +540,9 @@ def execute(session, collection, user, tableid, distinct, count_only, field_spec query = query.order_by(*order_by_exprs).offset(offset) if limit: query = query.limit(limit) -# query.groupBy('catalogNumber') + if series: + return {'results': list(query.group_by('catalognumber'))} + else: return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, @@ -568,6 +571,8 @@ def build_query(session, collection, user, tableid, field_specs, replace_nulls = if True, replace null values with "" distinct = if True, group by all display fields, and return all record IDs associated with a row + + series = (only for CO) if True, group by all catalog number in a same range, and return all record IDs associated with a row """ model = models.models_by_tableid[tableid] id_field = getattr(model, model._id) diff --git a/specifyweb/stored_queries/views.py b/specifyweb/stored_queries/views.py index 30499a48d36..493b1e0fca1 100644 --- a/specifyweb/stored_queries/views.py +++ b/specifyweb/stored_queries/views.py @@ -79,6 +79,7 @@ def query(request, id): with models.session_context() as session: sp_query = session.query(models.SpQuery).get(int(id)) distinct = sp_query.selectDistinct + series = sp_query.selectSeries tableid = sp_query.contextTableId count_only = sp_query.countOnly @@ -86,7 +87,7 @@ def query(request, id): for field in sorted(sp_query.fields, key=lambda field: field.position)] data = execute(session, request.specify_collection, request.specify_user, - tableid, distinct, count_only, field_specs, limit, offset) + tableid, distinct, series, count_only, field_specs, limit, offset) return HttpResponse(toJson(data), content_type='application/json') From e5a2d5080f4d422a6b52365193b4604b133cdeb3 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 11:30:39 -0700 Subject: [PATCH 14/59] Group by before limit --- specifyweb/stored_queries/execution.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 6ee3a7482c0..fa705613347 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -537,12 +537,15 @@ def execute(session, collection, user, tableid, distinct, series, count_only, fi return {'count': query.count()} else: logger.debug("order by: %s", order_by_exprs) + + if series: + query = query.group_by('catalognumber') + query = query.order_by(*order_by_exprs).offset(offset) + if limit: query = query.limit(limit) - if series: - return {'results': list(query.group_by('catalognumber'))} - else: + return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, From d6f3ff91d620f92a47e455d701c9ab3685461d65 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 12:53:49 -0700 Subject: [PATCH 15/59] Add comment --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index fa705613347..006bee7a0cc 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -575,7 +575,7 @@ def build_query(session, collection, user, tableid, field_specs, distinct = if True, group by all display fields, and return all record IDs associated with a row - series = (only for CO) if True, group by all catalog number in a same range, and return all record IDs associated with a row + series = (only for CO) if True, group by all display fields. Group catalog numbers that fall within the same range together. Return all record IDs associated with a row. """ model = models.models_by_tableid[tableid] id_field = getattr(model, model._id) From fed542f02e2d646325c54259af852d03c1d418a1 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 13:19:18 -0700 Subject: [PATCH 16/59] Add series logic --- specifyweb/stored_queries/execution.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 006bee7a0cc..be86edfa412 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -531,7 +531,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, fi "Build and execute a query, returning the results as a data structure for json serialization" set_group_concat_max_len(session) - query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, formatauditobjs=formatauditobjs, distinct=distinct) + query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, formatauditobjs=formatauditobjs, distinct=distinct, series=series) if count_only: return {'count': query.count()} @@ -549,7 +549,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, fi return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, - recordsetid=None, replace_nulls=False, formatauditobjs=False, distinct=False, implicit_or=True): + recordsetid=None, replace_nulls=False, formatauditobjs=False, distinct=False, series=False, implicit_or=True): """Build a sqlalchemy query using the QueryField objects given by field_specs. @@ -587,7 +587,7 @@ def build_query(session, collection, user, tableid, field_specs, query = QueryConstruct( collection=collection, objectformatter=ObjectFormatter(collection, user, replace_nulls), - query=session.query(func.group_concat(id_field.distinct(), separator=',')) if distinct else session.query(id_field), + query=session.query(func.group_concat(id_field.distinct(), separator=',')) if distinct or series else session.query(id_field), ) tables_to_read = set([ @@ -648,5 +648,8 @@ def build_query(session, collection, user, tableid, field_specs, if distinct: query = group_by_displayed_fields(query, selected_fields) + if series: + query = group_by_displayed_fields(query, selected_fields) + logger.debug("query: %s", query.query) return query.query, order_by_exprs From a1e37459236cf50faa222295cdcc28cb215083b2 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 24 May 2024 13:50:28 -0700 Subject: [PATCH 17/59] Start work on concat cat number --- specifyweb/stored_queries/execution.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index be86edfa412..238f8ef50d4 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -616,6 +616,7 @@ def build_query(session, collection, user, tableid, field_specs, selected_fields = [] predicates_by_field = defaultdict(list) #augment_field_specs(field_specs, formatauditobjs) + catalog_number_field = None for fs in field_specs: sort_type = SORT_TYPES[fs.sort_type] @@ -624,6 +625,9 @@ def build_query(session, collection, user, tableid, field_specs, formatted_field = query.objectformatter.fieldformat(fs, field) query = query.add_columns(formatted_field) selected_fields.append(formatted_field) + + if hasattr(field, 'key') and field.key == 'CatalogNumber': + catalog_number_field = formatted_field if sort_type is not None: order_by_exprs.append(sort_type(field)) @@ -645,11 +649,23 @@ def build_query(session, collection, user, tableid, field_specs, where = reduce(sql.and_, (p for ps in predicates_by_field.values() for p in ps)) query = query.filter(where) + if series: + selected_fields_without_cat_number = [] + for field in selected_fields: + if hasattr(field, 'clause') and hasattr(field.clause, 'key') and field.clause.key == 'CatalogNumber': + continue + selected_fields_without_cat_number.append(field) + # need to group concat only cat number that follow each other + if catalog_number_field is not None: + query = query.add_columns( + func.group_concat(catalog_number_field, separator=',') + ) + if distinct: query = group_by_displayed_fields(query, selected_fields) if series: - query = group_by_displayed_fields(query, selected_fields) + query = group_by_displayed_fields(query, selected_fields_without_cat_number) logger.debug("query: %s", query.query) return query.query, order_by_exprs From 33f43bf7a8f83ec1811231c8aaa82adf3e44b2d1 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 28 May 2024 08:12:13 -0700 Subject: [PATCH 18/59] Notes to concat cat number --- specifyweb/stored_queries/execution.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 238f8ef50d4..da93992def2 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -539,13 +539,30 @@ def execute(session, collection, user, tableid, distinct, series, count_only, fi logger.debug("order by: %s", order_by_exprs) if series: - query = query.group_by('catalognumber') + query = query.order_by('catalognumber') query = query.order_by(*order_by_exprs).offset(offset) if limit: query = query.limit(limit) + # newListQuery = [] + # if series: + # resultsFormList = list(query) + # # do some reogranization on the rows here + # for cat, index in resultsFormList actually need to go over the string of cat num and break it it will laways be the last column + # catRange = '' + # idRange = [] + # if index === 0: + # catRange.push(cat.cat) + # idRnage.push(cat.id) + # else cat === catRange.at(-1) + 1 ? catRange.push(cat) (// need catNum at idx 0 and then jst chnage the second since a rnage) & idRnage.push(cat.id) : + # 1) newListQuery.push(catRange, idRange, cat.otherfields) + # 2) catRange need to be cleared as well as idRange + # 3) push new cat and id to start again + + # return resultsFormList + return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, @@ -626,7 +643,7 @@ def build_query(session, collection, user, tableid, field_specs, query = query.add_columns(formatted_field) selected_fields.append(formatted_field) - if hasattr(field, 'key') and field.key == 'CatalogNumber': + if hasattr(field, 'key') and field.key.lower() == 'catalognumber': catalog_number_field = formatted_field if sort_type is not None: @@ -655,7 +672,7 @@ def build_query(session, collection, user, tableid, field_specs, if hasattr(field, 'clause') and hasattr(field.clause, 'key') and field.clause.key == 'CatalogNumber': continue selected_fields_without_cat_number.append(field) - # need to group concat only cat number that follow each other + if catalog_number_field is not None: query = query.add_columns( func.group_concat(catalog_number_field, separator=',') From d653b76380d925f02d6b428af46ff1603cf48ba1 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:45:38 -0700 Subject: [PATCH 19/59] Group cat number together test --- specifyweb/stored_queries/execution.py | 43 ++++++++++++++++---------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index da93992def2..0d2b682db39 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -546,22 +546,33 @@ def execute(session, collection, user, tableid, distinct, series, count_only, fi if limit: query = query.limit(limit) - # newListQuery = [] - # if series: - # resultsFormList = list(query) - # # do some reogranization on the rows here - # for cat, index in resultsFormList actually need to go over the string of cat num and break it it will laways be the last column - # catRange = '' - # idRange = [] - # if index === 0: - # catRange.push(cat.cat) - # idRnage.push(cat.id) - # else cat === catRange.at(-1) + 1 ? catRange.push(cat) (// need catNum at idx 0 and then jst chnage the second since a rnage) & idRnage.push(cat.id) : - # 1) newListQuery.push(catRange, idRange, cat.otherfields) - # 2) catRange need to be cleared as well as idRange - # 3) push new cat and id to start again - - # return resultsFormList + def is_consecutive(a, b): + return int(b) == int(a) + 1 + newListQuery = [] + if series: + resultsFormList = list(query) + + for item in resultsFormList: + #need to dertermine which item is the cat number and which is the id than group + fields = item[1:] # Get all fields except the first one + catalog_numbers = item[0].split(',') + if len(catalog_numbers) == 1: + newListQuery.append(item) + continue + + grouped_numbers = [catalog_numbers[0]] + for i in range(1, len(catalog_numbers)): + if is_consecutive(catalog_numbers[i - 1], catalog_numbers[i]): + grouped_numbers[-1] = f"{grouped_numbers[-1]},{catalog_numbers[i]}" + else: + newListQuery.append((grouped_numbers.pop(), *fields)) + grouped_numbers.append(catalog_numbers[i]) + + if grouped_numbers: + newListQuery.append((grouped_numbers.pop(), *fields)) + + print(newListQuery) + return {'results': list(query)} From f4cc35d7b62f9eecb0ddecfb3e4c94e49a31aa25 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 30 Jan 2025 11:49:40 -0600 Subject: [PATCH 20/59] implement back-end changes to meet Series QB features --- specifyweb/stored_queries/execution.py | 171 +++++++++++++++------- specifyweb/stored_queries/group_concat.py | 15 +- 2 files changed, 126 insertions(+), 60 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index ce934ea4a7f..31152854bec 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -26,6 +26,7 @@ from ..permissions.permissions import check_table_permissions from ..specify.auditlog import auditlog from ..specify.models import Loan, Loanpreparation, Loanreturnpreparation, Taxontreedef +from specifyweb.specify.utils import log_sqlalchemy_query logger = logging.getLogger(__name__) @@ -130,7 +131,6 @@ def filter_by_collection(model, query, collection): return query - EphemeralField = namedtuple('EphemeralField', "stringId isRelFld operStart startValue isNot isDisplay sortType formatName isStrict") def field_specs_from_json(json_fields): @@ -405,7 +405,7 @@ def run_ephemeral_query(collection, user, spquery): offset = spquery.get('offset', 0) recordsetid = spquery.get('recordsetid', None) distinct = spquery['selectdistinct'] - series = spquery['selectseries'] + series = spquery.get('selectseries', None) tableid = spquery['contexttableid'] count_only = spquery['countonly'] try: @@ -546,57 +546,35 @@ def return_loan_preps(collection, user, agent, data): ]) return to_return -def execute(session, collection, user, tableid, distinct, series, count_only, field_specs, limit, offset, recordsetid=None, formatauditobjs=False): +def execute(session, collection, user, tableid, distinct, series, count_only, + field_specs, limit, offset, recordsetid=None, formatauditobjs=False): "Build and execute a query, returning the results as a data structure for json serialization" set_group_concat_max_len(session.connection()) - query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, formatauditobjs=formatauditobjs, distinct=distinct, series=series) + query, order_by_exprs = build_query(session, collection, user, tableid, field_specs, recordsetid=recordsetid, + formatauditobjs=formatauditobjs, distinct=distinct, series=series) if count_only: return {'count': query.count()} else: logger.debug("order by: %s", order_by_exprs) - - if series: + if series: # maybe add - and catalog_number_field exists, and id_field doen't exist query = query.order_by('catalognumber') - + query = query.order_by(*order_by_exprs).offset(offset) - + if limit: query = query.limit(limit) - def is_consecutive(a, b): - return int(b) == int(a) + 1 - newListQuery = [] - if series: - resultsFormList = list(query) - - for item in resultsFormList: - #need to dertermine which item is the cat number and which is the id than group - fields = item[1:] # Get all fields except the first one - catalog_numbers = item[0].split(',') - if len(catalog_numbers) == 1: - newListQuery.append(item) - continue - - grouped_numbers = [catalog_numbers[0]] - for i in range(1, len(catalog_numbers)): - if is_consecutive(catalog_numbers[i - 1], catalog_numbers[i]): - grouped_numbers[-1] = f"{grouped_numbers[-1]},{catalog_numbers[i]}" - else: - newListQuery.append((grouped_numbers.pop(), *fields)) - grouped_numbers.append(catalog_numbers[i]) - - if grouped_numbers: - newListQuery.append((grouped_numbers.pop(), *fields)) - - print(newListQuery) - + if series: + return {'results': series_post_query(query)} + log_sqlalchemy_query(query) # Debugging return {'results': list(query)} def build_query(session, collection, user, tableid, field_specs, - recordsetid=None, replace_nulls=False, formatauditobjs=False, distinct=False, series=False, implicit_or=True): + recordsetid=None, replace_nulls=False, formatauditobjs=False, + distinct=False, series=False, implicit_or=True): """Build a sqlalchemy query using the QueryField objects given by field_specs. @@ -622,19 +600,30 @@ def build_query(session, collection, user, tableid, field_specs, distinct = if True, group by all display fields, and return all record IDs associated with a row - series = (only for CO) if True, group by all display fields. Group catalog numbers that fall within the same range together. Return all record IDs associated with a row. + series = (only for CO) if True, group by all display fields. + Group catalog numbers that fall within the same range together. + Return all record IDs associated with a row. """ model = models.models_by_tableid[tableid] id_field = getattr(model, model._id) + catalog_number_field = model.catalogNumber if hasattr(model, 'catalogNumber') else None field_specs = [apply_absolute_date(field_spec) for field_spec in field_specs] field_specs = [apply_specify_user_name(field_spec, user) for field_spec in field_specs] - + query_construct_query = None + if series and catalog_number_field: + query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=','), + func.group_concat(catalog_number_field.distinct(), separator=',')) + elif distinct: + query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=',')) + else: + query_construct_query = query_construct_query = session.query(id_field) + query = QueryConstruct( collection=collection, objectformatter=ObjectFormatter(collection, user, replace_nulls), - query=session.query(func.group_concat(id_field.distinct(), separator=',')) if distinct or series else session.query(id_field), + query=query_construct_query, ) tables_to_read = set([ @@ -662,17 +651,24 @@ def build_query(session, collection, user, tableid, field_specs, order_by_exprs = [] selected_fields = [] predicates_by_field = defaultdict(list) - #augment_field_specs(field_specs, formatauditobjs) - catalog_number_field = None + # augment_field_specs(field_specs, formatauditobjs) for fs in field_specs: sort_type = SORT_TYPES[fs.sort_type] + if series and fs.fieldspec.get_field().name.lower() == 'catalognumber': + continue + query, field, predicate = fs.add_to_query(query, formatauditobjs=formatauditobjs) if fs.display: formatted_field = query.objectformatter.fieldformat(fs, field) query = query.add_columns(formatted_field) selected_fields.append(formatted_field) + + if hasattr(field, 'key') and field.key.lower() == 'catalognumber': + catalog_number_field = formatted_field + + if hasattr(field, 'key') and field.key.lower() == 'catalognumber': catalog_number_field = formatted_field @@ -697,25 +693,88 @@ def build_query(session, collection, user, tableid, field_specs, query = query.filter(where) if series: - selected_fields_without_cat_number = [] - for field in selected_fields: - if hasattr(field, 'clause') and hasattr(field.clause, 'key') and field.clause.key == 'CatalogNumber': - continue - selected_fields_without_cat_number.append(field) - - if catalog_number_field is not None: - query = query.add_columns( - func.group_concat(catalog_number_field, separator=',') - ) - - if distinct: + query = group_by_displayed_fields(query, selected_fields, ignore_cat_num=True) + elif distinct: query = group_by_displayed_fields(query, selected_fields) - if series: - query = group_by_displayed_fields(query, selected_fields_without_cat_number) - internal_predicate = query.get_internal_filters() query = query.filter(internal_predicate) logger.warning("query: %s", query.query) return query.query, order_by_exprs + +def series_post_query(query, co_id_col_index = 0, co_cat_num_col_index = 1): + def process_group_by_result(group_by_query_result, id_col_index = 0, group_col_index = 1): + def find_consecutive_ranges(lst): + def group_consecutives(acc, x): + if not acc or acc[-1][-1] + 1 != x: + acc.append([x]) + else: + acc[-1].append(x) + return acc + + grouped = reduce(group_consecutives, lst, []) + + return [f"{g[0]:04d} - {g[-1]:04d}" if len(g) > 1 else f"{g[0]:04d}" for g in grouped] + + def parse_numbers(num_str): + return sorted(map(int, filter(None, map(str.strip, num_str.replace(',', ' ').split())))) + + def format_record(record): + id_part = record[id_col_index] + id_values = id_part.split(',') + + num_ranges = find_consecutive_ranges(parse_numbers(record[group_col_index])) + formatted_records = [[id_values[0]] + [num_ranges[0]] + list(record[2:])] if len(id_values) == 1 else [] + + if len(num_ranges) > 1: + for num_range in num_ranges[1:]: + formatted_records.append([id_values.pop()] + [num_range] + list(record[2:])) + + return formatted_records if formatted_records else [[id_part] + [num_ranges[0]] + list(record[2:])] + + formatted_records = [format_record(record[:]) for record in group_by_query_result] + + result = [item for sublist in formatted_records for item in sublist] + result.sort(key=lambda x: int(x[1].split(' - ')[0])) + + return result + + return process_group_by_result(list(query), co_id_col_index, co_cat_num_col_index) + +def series_post_query_test(query): # TODO: Remove after adding unit tests + + query_results = list(query) + series_query_results = [] + + input = [["0012,0013,0014", "SomeText1", "Vial"], + ["0015", "OtherText", "Vial"], + ["0016,", "AnotherText", "Vial"], + ["0017,0018", "SomeText2", "Vial"], + ["0020, 0021, 0022", "SomeText3", "Vial"]] + + output = [["0012 - 0014", "SomeText1", "Vial"], + ["0015", "OtherText", "Vial"], + ["0016", "AnotherText", "Vial"], + ["0017 - 0018", "SomeText2", "Vial"], + ["0020 - 0022", "SomeText3", "Vial"]] + + + input = [ + ["1,2,3", "0021,0022,0043", "SomeText1", "Vial"], + ["4", "0023", "OtherText", "Vial"], + ["5", "0024", "AnotherText", "Vial"], + ["6,7", "0025,0026", "SomeText2", "Vial"], + ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"] + ] + + output = [ + ["1,2", "0021 - 0022", "SomeText1", "Vial"], + ["4", "0023", "OtherText", "Vial"], + ["5", "0024", "AnotherText", "Vial"], + ["6,7", "0025 - 0026", "SomeText2", "Vial"], + ["8,9,10", "0027 - 0029", "SomeText3", "Vial"], + ["3", "0043", "SomeText1", "Vial"] + ] + + return series_query_results \ No newline at end of file diff --git a/specifyweb/stored_queries/group_concat.py b/specifyweb/stored_queries/group_concat.py index 1ed3a6ebc25..a6051408819 100644 --- a/specifyweb/stored_queries/group_concat.py +++ b/specifyweb/stored_queries/group_concat.py @@ -1,7 +1,6 @@ # Based on stackoverflow answer from Wolph: # https://stackoverflow.com/questions/19205850/how-do-i-write-a-group-concat-function-in-sqlalchemy -import re import sqlalchemy from sqlalchemy.sql import expression from sqlalchemy.ext import compiler @@ -43,8 +42,16 @@ def process_clause(idx): return expr, separator, order_by -def group_by_displayed_fields(query: QueryConstruct, fields): +def group_by_displayed_fields(query: QueryConstruct, fields, ignore_cat_num=False): for field in fields: + if ( + ignore_cat_num + and hasattr(field, "clause") + and field.clause is not None + and hasattr(field.clause, "key") + and field.clause.key == "CatalogNumber" + ): + continue query = query.group_by(field) - - return query + + return query \ No newline at end of file From 7a3707ea9f0b3c369e08447e995b7f1e0162ccee Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:37:30 -0800 Subject: [PATCH 21/59] Only allow series or distinct --- .../frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx | 2 ++ .../__tests__/__snapshots__/fromTree.test.ts.snap | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx index 4372fdc2a54..7a609c0c19e 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Toolbar.tsx @@ -48,6 +48,7 @@ export function QueryToolbar({ {queryText.series()} @@ -61,6 +62,7 @@ export function QueryToolbar({ {queryText.distinct()} diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/__tests__/__snapshots__/fromTree.test.ts.snap b/specifyweb/frontend/js_src/lib/components/QueryBuilder/__tests__/__snapshots__/fromTree.test.ts.snap index 16b955a071f..81fd3b4f871 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/__tests__/__snapshots__/fromTree.test.ts.snap +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/__tests__/__snapshots__/fromTree.test.ts.snap @@ -86,6 +86,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Los Angeles County\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, { @@ -160,6 +161,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Cabinet 1\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, { @@ -234,6 +236,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Carpiodes velifer\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, { @@ -308,6 +311,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Paleocene\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, { @@ -382,6 +386,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Cretaceous\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, { @@ -456,6 +461,7 @@ exports[`queryFromTree 1`] = ` "name": "Collection Object using \\"Plate\\"", "ordinal": 32767, "selectdistinct": false, + "selectseries": false, "specifyuser": "/api/specify/specifyuser/2/", }, ] From b8072073cb7f60b52362b012f0594db400f390cd Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 31 Jan 2025 09:17:47 -0600 Subject: [PATCH 22/59] remove duplicate code --- specifyweb/stored_queries/execution.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 31152854bec..b005587cc66 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -665,10 +665,6 @@ def build_query(session, collection, user, tableid, field_specs, selected_fields.append(formatted_field) - if hasattr(field, 'key') and field.key.lower() == 'catalognumber': - catalog_number_field = formatted_field - - if hasattr(field, 'key') and field.key.lower() == 'catalognumber': catalog_number_field = formatted_field From 55dbc0f989d22031eedab01b2f77f16b96d07a87 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 31 Jan 2025 16:04:02 -0600 Subject: [PATCH 23/59] better series_post_query function --- specifyweb/stored_queries/execution.py | 147 +++++++++++++++++-------- 1 file changed, 102 insertions(+), 45 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index b005587cc66..6e472512a4f 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -10,7 +10,7 @@ from django.conf import settings from django.db import transaction -from sqlalchemy import sql, orm, func, select +from sqlalchemy import sql, orm, func, select, text from sqlalchemy.sql.expression import asc, desc, insert, literal from specifyweb.stored_queries.group_concat import group_by_displayed_fields @@ -558,8 +558,8 @@ def execute(session, collection, user, tableid, distinct, series, count_only, return {'count': query.count()} else: logger.debug("order by: %s", order_by_exprs) - if series: # maybe add - and catalog_number_field exists, and id_field doen't exist - query = query.order_by('catalognumber') + if series: + order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) query = query.order_by(*order_by_exprs).offset(offset) @@ -613,8 +613,16 @@ def build_query(session, collection, user, tableid, field_specs, query_construct_query = None if series and catalog_number_field: - query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=','), - func.group_concat(catalog_number_field.distinct(), separator=',')) + query_construct_query = session.query( + func.group_concat( + func.concat( + id_field, + ':', + catalog_number_field + ), + separator='|' + ).label('co_id_catnum_paired_values') + ) elif distinct: query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=',')) else: @@ -664,7 +672,6 @@ def build_query(session, collection, user, tableid, field_specs, query = query.add_columns(formatted_field) selected_fields.append(formatted_field) - if hasattr(field, 'key') and field.key.lower() == 'catalognumber': catalog_number_field = formatted_field @@ -699,44 +706,43 @@ def build_query(session, collection, user, tableid, field_specs, logger.warning("query: %s", query.query) return query.query, order_by_exprs -def series_post_query(query, co_id_col_index = 0, co_cat_num_col_index = 1): - def process_group_by_result(group_by_query_result, id_col_index = 0, group_col_index = 1): - def find_consecutive_ranges(lst): - def group_consecutives(acc, x): - if not acc or acc[-1][-1] + 1 != x: - acc.append([x]) - else: - acc[-1].append(x) - return acc - - grouped = reduce(group_consecutives, lst, []) - - return [f"{g[0]:04d} - {g[-1]:04d}" if len(g) > 1 else f"{g[0]:04d}" for g in grouped] - - def parse_numbers(num_str): - return sorted(map(int, filter(None, map(str.strip, num_str.replace(',', ' ').split())))) - - def format_record(record): - id_part = record[id_col_index] - id_values = id_part.split(',') - - num_ranges = find_consecutive_ranges(parse_numbers(record[group_col_index])) - formatted_records = [[id_values[0]] + [num_ranges[0]] + list(record[2:])] if len(id_values) == 1 else [] - - if len(num_ranges) > 1: - for num_range in num_ranges[1:]: - formatted_records.append([id_values.pop()] + [num_range] + list(record[2:])) - - return formatted_records if formatted_records else [[id_part] + [num_ranges[0]] + list(record[2:])] - - formatted_records = [format_record(record[:]) for record in group_by_query_result] - - result = [item for sublist in formatted_records for item in sublist] - result.sort(key=lambda x: int(x[1].split(' - ')[0])) - - return result +def series_post_query(query, co_id_cat_num_pair_col_index=0): + """Transform the query results by removing the co_id:catnum pair column + and adding a co_id colum and formatted catnum range column. + Sort the results by the first catnum in the range.""" + log_sqlalchemy_query(query) # Debugging + + def group_consecutive_ranges(lst): + def group_consecutives(acc, x): + if not acc or int(acc[-1][-1][1]) + 1 != int(x[1]): + acc.append([x]) + else: + acc[-1].append(x) + return acc + + grouped = reduce(group_consecutives, lst, []) + return [ + (','.join([x[0] for x in group]), f"{group[0][1]} - {group[-1][1]}" if len(group) > 1 else f"{group[0][1]}") + for group in grouped + ] + + def process_row(row): + co_id_cat_num_consecutive_pairs = group_consecutive_ranges( + sorted( + (pair.split(':') for pair in row[co_id_cat_num_pair_col_index].split(',')), + key=lambda x: int(x[1]) + ) + ) - return process_group_by_result(list(query), co_id_col_index, co_cat_num_col_index) + return [ + [co_id, cat_num_series] + list( + list(row[1:]) if co_id_cat_num_pair_col_index == 0 + else list(row[:co_id_cat_num_pair_col_index]) + list(row[co_id_cat_num_pair_col_index + 1:]) + ) + for co_id, cat_num_series in co_id_cat_num_consecutive_pairs + ] + + return [item for sublist in map(process_row, list(query)) for item in sublist] def series_post_query_test(query): # TODO: Remove after adding unit tests @@ -763,7 +769,6 @@ def series_post_query_test(query): # TODO: Remove after adding unit tests ["6,7", "0025,0026", "SomeText2", "Vial"], ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"] ] - output = [ ["1,2", "0021 - 0022", "SomeText1", "Vial"], ["4", "0023", "OtherText", "Vial"], @@ -773,4 +778,56 @@ def series_post_query_test(query): # TODO: Remove after adding unit tests ["3", "0043", "SomeText1", "Vial"] ] - return series_query_results \ No newline at end of file + input = [ + ["1,2,3", "0021,0022,0043", "SomeText1", "Vial"], + ["4", "0023", "OtherText", "Vial"], + ["5", "0024", "AnotherText", "Vial"], + ["6,7", "0025,0026", "SomeText2", "Vial"], + ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"] + ] + output = [ + ["1,2", "21 - 22", "SomeText1", "Vial"], + ["4", "23", "OtherText", "Vial"], + ["5", "24", "AnotherText", "Vial"], + ["6,7", "25 - 26", "SomeText2", "Vial"], + ["8,9,10", "27 - 29", "SomeText3", "Vial"], + ["3", "43", "SomeText1", "Vial"] + ] + + input = [ + ["16586,28543", "000061109,000061110", "Keebaugh"], + ["223224,223707,223712,223713,223792,223880,223881,223882,223883,223884,223885,223886,223887,223888,223889,224121,224122,224123,224124,224139,224140,224141,224142,224143,224144,224145,224149,224158,224159,224160,224161,224200,224201,224202,224203,224204,224205,224206,224207,224230,224231,224232,224233,224234,224235,224236,224237,224238,224239,224240,224255,224256,224257,224258,224259,224260,224261,224554,224555,224556,224557,224561,224652,224653,224654,224655,224656,224657,224658,224659,224660,224661,224662,224663,224664,224665,224666,224667,224668,224669,224670,224671,224672,224673,224674,224675,224676,224677,224678,224679,224680,224681,224746,224747,224748,224749,224750,224751,224752,224753,224754,224755,224756,224757,224758,224759,224760,224761,224762,224763,224764,224765,224766,224767,224768,224769,224770,224771,224772,224773,224774,224775,224776,224777,224917,224918,224919,224920,224921,224922,224923,224924,224925,224926,224927,224928,224929,224930,224931,224932,224933,224934,224935,224936,224937,224938,224939,224940,224941,224942,224943,224944,224945,224946,224947,224948,224949,225538,225539,225540,225541,225542,225543,225544,225545,225546,225547,225548,225549,225550,225551,225552,225553,225554,225555,225556,225557,225558,225559,225560,225561,225562,225563,225564,225565,225566,225567,225568,225569,225570,225571,225572,225573,225574,225575,225576,225577,225578,225579,225580,225581,225582,225583,225584,225585,225586,225587,225588,225589,225590,225591,225592,225593,225594,225595,225596,225597,225598,225599,225600,225601,225602,225603,225604,225605,225606,225607,225608,225609,225610,225611,225612,225613,225614,225615,225616,225617,225618,225619,225620,225621,225622,225623,225624,225625,225626,225627,225628,225629,225630,225631,225632,225633,225634,225936,225937,225938,225939,225940,225941,225942,225943,225944,225945,225946,225947,225948,225949,225950,225951,225952,225953,225954,225955,225956,225957,225958,225959,225960,225961,225962,225963,225964,225965,225966,225967,227169,227170,227171,227172,227173,227174,227175,227176,227177,227178,227179,227180,227181,227182,227183,227184,227185,227186,227187,227188,227189,227190,227191,227192,227193,227194,227195,227196,227197,227198,227199,227200,227298,227299,227300,227301,227302,227303,227304,227305,227306,227307,227308,227309,227310,227311,227312,227313,227314,227315,227316,227317,227318,227319,227320,227321,227322,227323,227324,227325,227326,227327,227328,227329,227330,227427,227428,227429,227430,227431,227432,227433,227434,227435,227436,227437,227438,227439,227440,227441,227442,227443,227444,227445,227446,227447,227448,227449,227450,227451,227452,227453,227454,227455,227456,227457,227458,227560,227561,227562,227563,227564,227565,227566,227567,227568,227569,227570,227571,227572,227573,227574,227575,227576,227577,227578,227579,227580,227581,227582,227583,227584,227585,227586,227587,227588,227589,227590,227591,227656,227657,227658,227659,227660,227661,227662,227663,227664,227665,227666,227667,227668,227669,227670,227671,227672,227673,227674,227675,227676,227677,227678,227679,227680,227681,227682,227683,227684,227685,227686,227687", + "000061053,000061054,000061055,000061056,000061057,000061058,000061059,000061060,000061061,000061062,000061063,000061064,000061065,000061066,000061067,000061068,000061069,000061070,000061071,000061072,000061073,000061074,000061075,000061076,000061077,000061078,000061079,000061080,000061081,000061082,000061083,000061084,000061085,000061086,000061087,000061088,000061089,000061090,000061091,000061092,000061093,000061094,000061095,000061096,000061097,000061098,000061099,000061100,000061101,000061102,000061103,000061104,000062656,000101392,000101571,000101645,000101672,000101872,000101873,000101874,000101875,000101888,000102018,000102019,000102020,000102110,000102420,000102421,000102422,000102423,000102424,000102425,000102426,000102427,000102428,000102429,000102491,000113183,000113689,000113751,000114066,000114140,000114207,000118367,000118559,000118643,000119023,000119047,000119162,000119242,000119342,000119428,000119551,000119552,000119553,000119554,000119555,000119556,000119557,000119558,000119559,000119560,000119561,000119562,000119563,000119564,000119565,000119566,000119567,000119568,000119569,000119570,000119571,000119572,000119573,000119574,000119575,000119576,000119577,000119578,000119579,000119580,000119581,000119582,000119583,000119584,000119585,000119586,000119587,000119588,000119589,000119590,000119591,000119592,000119593,000119594,000119595,000119596,000119597,000119598,000119599,000119600,000119601,000119602,000119603,000119604,000119605,000119606,000119607,000119608,000119609,000119610,000119611,000119612,000119613,000119614,000119615,000119616,000119617,000119618,000119619,000119620,000119621,000119622,000119623,000119624,000119625,000119626,000119627,000119628,000119629,000119630,000119631,000119632,000119633,000119634,000119635,000119636,000119637,000119638,000119639,000119640,000119641,000119642,000119643,000119644,000119645,000119647,000119648,000119649,000119650,000119651,000119652,000119653,000119654,000119655,000119656,000119657,000119658,000119659,000119660,000119661,000119662,000119663,000119664,000119665,000119666,000119667,000119668,000119669,000119670,000119671,000119672,000119673,000119674,000119675,000119676,000119677,000119678,000119679,000119680,000119681,000119682,000119683,000119684,000119685,000119686,000119687,000119688,000119689,000119690,000119691,000119692,000119693,000119694,000119695,000119696,000119697,000119698,000119699,000119700,000119701,000119702,000119703,000119704,000119705,000119706,000119707,000119708,000119709,000119710,000119711,000119712,000119713,000119714,000119715,000119716,000119717,000119718,000119719,000119720,000119721,000119722,000119723,000119724,000119725,000119726,000119727,000119728,000119729,000119730,000119731,000119732,000119733,000119734,000119735,000119736,000119737,000119738,000119739,000119740,000119741,000119742,000119743,000119744,000119745,000119746,000119747,000119748,000119749,000119750,000119751,000119752,000119753,000119754,000119755,000119756,000119757,000119758,000119759,000119760,000119761,000119762,000119763,000119764,000119765,000119766,000119767,000119768,000119769,000119770,000119771,000119772,000119773,000119774,000119775,000119776,000119777,000119778,000119779,000119780,000119781,000119782,000119783,000119784,000119785,000119786,000119787,000119788,000119789,000119790,000119791,000119792,000119793,000119794,000119795,000119796,000119797,000119798,000119799,000119800,000119801,000119802,000119803,000119804,000119805,000119806,000119807,000119808,000119809,000119810,000119811,000119812,000119813,000119814,000119815,000119816,000119817,000119818,000119819,000119820,000119821,000119822,000119823,000119824,000119825,000119826,000119827,000119828,000119829,000119830,000119831,000119832,000119833,000119834,000119835,000119836,000119837,000119838,000119839,000119840,000119841,000119842,000119843,000119844,000119845,000119846,000119847,000119848,000119849,000119850,000119851,000119852,000119853,000119854,000119855,000119856,000119857,000119858,000119859,000119860,000119861,000119862,000119863,000119864,000119865,000119866,000119867,000119868,000119869,000119870,000119871,000119872,000119873,000119874,000119875,000119876,000119877,000119878,000119879,000119880,000119881,000119882,000119883,000119884,000119885,000119886,000119887,000119888,000119889,000119890,000119891,000119892,000119893,000119894,000119895,000119896,000119897,000119898,000119899,000119900,000119901,000119902,000119903,000119904,000119905,000119906", + "Blom" + ] + ] + output = [ + ["51518,51519,51520,51521", "9423 - 9426", "Fernandes"], + ["51718", "9427 - 9500", "Barbosa"], + ["51719", "9501","Admin"] + ] + desired_output = [ + ["51518 - 51521", "9423 - 9426", "Fernandes"], + ["51718", "9427 - 9500", "Barbosa"], + ["51719", "9501", "Admin"] + ] + + co_id_cat_num_pair_col_index = 0 + input = [ + ["1:0021,2:0022,3:0043", "SomeText1", "Vial"], + ["4:0023", "OtherText", "Vial"], + ["5:0024", "AnotherText", "Vial"], + ["6:0025,7:0026", "SomeText2", "Vial"], + ["8:0027,9:0028,10:0029", "SomeText3", "Vial"] + ] + output = [ + ["1,2", "21 - 22", "SomeText1", "Vial"], + ["4", "23", "OtherText", "Vial"], + ["5", "24", "AnotherText", "Vial"], + ["6,7", "25 - 26", "SomeText2", "Vial"], + ["8,9,10", "27 - 29", "SomeText3", "Vial"], + ["3", "43", "SomeText1", "Vial"] + ] + + + return series_query_results From 95d4d6089542ddaab27f8aab3efe2d0566044c2e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 31 Jan 2025 16:22:40 -0600 Subject: [PATCH 24/59] cleanup --- specifyweb/stored_queries/execution.py | 90 +------------------------- 1 file changed, 1 insertion(+), 89 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 6e472512a4f..f41983b0d7e 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -557,10 +557,10 @@ def execute(session, collection, user, tableid, distinct, series, count_only, if count_only: return {'count': query.count()} else: - logger.debug("order by: %s", order_by_exprs) if series: order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) + logger.debug("order by: %s", order_by_exprs) query = query.order_by(*order_by_exprs).offset(offset) if limit: @@ -743,91 +743,3 @@ def process_row(row): ] return [item for sublist in map(process_row, list(query)) for item in sublist] - -def series_post_query_test(query): # TODO: Remove after adding unit tests - - query_results = list(query) - series_query_results = [] - - input = [["0012,0013,0014", "SomeText1", "Vial"], - ["0015", "OtherText", "Vial"], - ["0016,", "AnotherText", "Vial"], - ["0017,0018", "SomeText2", "Vial"], - ["0020, 0021, 0022", "SomeText3", "Vial"]] - - output = [["0012 - 0014", "SomeText1", "Vial"], - ["0015", "OtherText", "Vial"], - ["0016", "AnotherText", "Vial"], - ["0017 - 0018", "SomeText2", "Vial"], - ["0020 - 0022", "SomeText3", "Vial"]] - - - input = [ - ["1,2,3", "0021,0022,0043", "SomeText1", "Vial"], - ["4", "0023", "OtherText", "Vial"], - ["5", "0024", "AnotherText", "Vial"], - ["6,7", "0025,0026", "SomeText2", "Vial"], - ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"] - ] - output = [ - ["1,2", "0021 - 0022", "SomeText1", "Vial"], - ["4", "0023", "OtherText", "Vial"], - ["5", "0024", "AnotherText", "Vial"], - ["6,7", "0025 - 0026", "SomeText2", "Vial"], - ["8,9,10", "0027 - 0029", "SomeText3", "Vial"], - ["3", "0043", "SomeText1", "Vial"] - ] - - input = [ - ["1,2,3", "0021,0022,0043", "SomeText1", "Vial"], - ["4", "0023", "OtherText", "Vial"], - ["5", "0024", "AnotherText", "Vial"], - ["6,7", "0025,0026", "SomeText2", "Vial"], - ["8,9,10", "0027,0028,0029", "SomeText3", "Vial"] - ] - output = [ - ["1,2", "21 - 22", "SomeText1", "Vial"], - ["4", "23", "OtherText", "Vial"], - ["5", "24", "AnotherText", "Vial"], - ["6,7", "25 - 26", "SomeText2", "Vial"], - ["8,9,10", "27 - 29", "SomeText3", "Vial"], - ["3", "43", "SomeText1", "Vial"] - ] - - input = [ - ["16586,28543", "000061109,000061110", "Keebaugh"], - ["223224,223707,223712,223713,223792,223880,223881,223882,223883,223884,223885,223886,223887,223888,223889,224121,224122,224123,224124,224139,224140,224141,224142,224143,224144,224145,224149,224158,224159,224160,224161,224200,224201,224202,224203,224204,224205,224206,224207,224230,224231,224232,224233,224234,224235,224236,224237,224238,224239,224240,224255,224256,224257,224258,224259,224260,224261,224554,224555,224556,224557,224561,224652,224653,224654,224655,224656,224657,224658,224659,224660,224661,224662,224663,224664,224665,224666,224667,224668,224669,224670,224671,224672,224673,224674,224675,224676,224677,224678,224679,224680,224681,224746,224747,224748,224749,224750,224751,224752,224753,224754,224755,224756,224757,224758,224759,224760,224761,224762,224763,224764,224765,224766,224767,224768,224769,224770,224771,224772,224773,224774,224775,224776,224777,224917,224918,224919,224920,224921,224922,224923,224924,224925,224926,224927,224928,224929,224930,224931,224932,224933,224934,224935,224936,224937,224938,224939,224940,224941,224942,224943,224944,224945,224946,224947,224948,224949,225538,225539,225540,225541,225542,225543,225544,225545,225546,225547,225548,225549,225550,225551,225552,225553,225554,225555,225556,225557,225558,225559,225560,225561,225562,225563,225564,225565,225566,225567,225568,225569,225570,225571,225572,225573,225574,225575,225576,225577,225578,225579,225580,225581,225582,225583,225584,225585,225586,225587,225588,225589,225590,225591,225592,225593,225594,225595,225596,225597,225598,225599,225600,225601,225602,225603,225604,225605,225606,225607,225608,225609,225610,225611,225612,225613,225614,225615,225616,225617,225618,225619,225620,225621,225622,225623,225624,225625,225626,225627,225628,225629,225630,225631,225632,225633,225634,225936,225937,225938,225939,225940,225941,225942,225943,225944,225945,225946,225947,225948,225949,225950,225951,225952,225953,225954,225955,225956,225957,225958,225959,225960,225961,225962,225963,225964,225965,225966,225967,227169,227170,227171,227172,227173,227174,227175,227176,227177,227178,227179,227180,227181,227182,227183,227184,227185,227186,227187,227188,227189,227190,227191,227192,227193,227194,227195,227196,227197,227198,227199,227200,227298,227299,227300,227301,227302,227303,227304,227305,227306,227307,227308,227309,227310,227311,227312,227313,227314,227315,227316,227317,227318,227319,227320,227321,227322,227323,227324,227325,227326,227327,227328,227329,227330,227427,227428,227429,227430,227431,227432,227433,227434,227435,227436,227437,227438,227439,227440,227441,227442,227443,227444,227445,227446,227447,227448,227449,227450,227451,227452,227453,227454,227455,227456,227457,227458,227560,227561,227562,227563,227564,227565,227566,227567,227568,227569,227570,227571,227572,227573,227574,227575,227576,227577,227578,227579,227580,227581,227582,227583,227584,227585,227586,227587,227588,227589,227590,227591,227656,227657,227658,227659,227660,227661,227662,227663,227664,227665,227666,227667,227668,227669,227670,227671,227672,227673,227674,227675,227676,227677,227678,227679,227680,227681,227682,227683,227684,227685,227686,227687", - "000061053,000061054,000061055,000061056,000061057,000061058,000061059,000061060,000061061,000061062,000061063,000061064,000061065,000061066,000061067,000061068,000061069,000061070,000061071,000061072,000061073,000061074,000061075,000061076,000061077,000061078,000061079,000061080,000061081,000061082,000061083,000061084,000061085,000061086,000061087,000061088,000061089,000061090,000061091,000061092,000061093,000061094,000061095,000061096,000061097,000061098,000061099,000061100,000061101,000061102,000061103,000061104,000062656,000101392,000101571,000101645,000101672,000101872,000101873,000101874,000101875,000101888,000102018,000102019,000102020,000102110,000102420,000102421,000102422,000102423,000102424,000102425,000102426,000102427,000102428,000102429,000102491,000113183,000113689,000113751,000114066,000114140,000114207,000118367,000118559,000118643,000119023,000119047,000119162,000119242,000119342,000119428,000119551,000119552,000119553,000119554,000119555,000119556,000119557,000119558,000119559,000119560,000119561,000119562,000119563,000119564,000119565,000119566,000119567,000119568,000119569,000119570,000119571,000119572,000119573,000119574,000119575,000119576,000119577,000119578,000119579,000119580,000119581,000119582,000119583,000119584,000119585,000119586,000119587,000119588,000119589,000119590,000119591,000119592,000119593,000119594,000119595,000119596,000119597,000119598,000119599,000119600,000119601,000119602,000119603,000119604,000119605,000119606,000119607,000119608,000119609,000119610,000119611,000119612,000119613,000119614,000119615,000119616,000119617,000119618,000119619,000119620,000119621,000119622,000119623,000119624,000119625,000119626,000119627,000119628,000119629,000119630,000119631,000119632,000119633,000119634,000119635,000119636,000119637,000119638,000119639,000119640,000119641,000119642,000119643,000119644,000119645,000119647,000119648,000119649,000119650,000119651,000119652,000119653,000119654,000119655,000119656,000119657,000119658,000119659,000119660,000119661,000119662,000119663,000119664,000119665,000119666,000119667,000119668,000119669,000119670,000119671,000119672,000119673,000119674,000119675,000119676,000119677,000119678,000119679,000119680,000119681,000119682,000119683,000119684,000119685,000119686,000119687,000119688,000119689,000119690,000119691,000119692,000119693,000119694,000119695,000119696,000119697,000119698,000119699,000119700,000119701,000119702,000119703,000119704,000119705,000119706,000119707,000119708,000119709,000119710,000119711,000119712,000119713,000119714,000119715,000119716,000119717,000119718,000119719,000119720,000119721,000119722,000119723,000119724,000119725,000119726,000119727,000119728,000119729,000119730,000119731,000119732,000119733,000119734,000119735,000119736,000119737,000119738,000119739,000119740,000119741,000119742,000119743,000119744,000119745,000119746,000119747,000119748,000119749,000119750,000119751,000119752,000119753,000119754,000119755,000119756,000119757,000119758,000119759,000119760,000119761,000119762,000119763,000119764,000119765,000119766,000119767,000119768,000119769,000119770,000119771,000119772,000119773,000119774,000119775,000119776,000119777,000119778,000119779,000119780,000119781,000119782,000119783,000119784,000119785,000119786,000119787,000119788,000119789,000119790,000119791,000119792,000119793,000119794,000119795,000119796,000119797,000119798,000119799,000119800,000119801,000119802,000119803,000119804,000119805,000119806,000119807,000119808,000119809,000119810,000119811,000119812,000119813,000119814,000119815,000119816,000119817,000119818,000119819,000119820,000119821,000119822,000119823,000119824,000119825,000119826,000119827,000119828,000119829,000119830,000119831,000119832,000119833,000119834,000119835,000119836,000119837,000119838,000119839,000119840,000119841,000119842,000119843,000119844,000119845,000119846,000119847,000119848,000119849,000119850,000119851,000119852,000119853,000119854,000119855,000119856,000119857,000119858,000119859,000119860,000119861,000119862,000119863,000119864,000119865,000119866,000119867,000119868,000119869,000119870,000119871,000119872,000119873,000119874,000119875,000119876,000119877,000119878,000119879,000119880,000119881,000119882,000119883,000119884,000119885,000119886,000119887,000119888,000119889,000119890,000119891,000119892,000119893,000119894,000119895,000119896,000119897,000119898,000119899,000119900,000119901,000119902,000119903,000119904,000119905,000119906", - "Blom" - ] - ] - output = [ - ["51518,51519,51520,51521", "9423 - 9426", "Fernandes"], - ["51718", "9427 - 9500", "Barbosa"], - ["51719", "9501","Admin"] - ] - desired_output = [ - ["51518 - 51521", "9423 - 9426", "Fernandes"], - ["51718", "9427 - 9500", "Barbosa"], - ["51719", "9501", "Admin"] - ] - - co_id_cat_num_pair_col_index = 0 - input = [ - ["1:0021,2:0022,3:0043", "SomeText1", "Vial"], - ["4:0023", "OtherText", "Vial"], - ["5:0024", "AnotherText", "Vial"], - ["6:0025,7:0026", "SomeText2", "Vial"], - ["8:0027,9:0028,10:0029", "SomeText3", "Vial"] - ] - output = [ - ["1,2", "21 - 22", "SomeText1", "Vial"], - ["4", "23", "OtherText", "Vial"], - ["5", "24", "AnotherText", "Vial"], - ["6,7", "25 - 26", "SomeText2", "Vial"], - ["8,9,10", "27 - 29", "SomeText3", "Vial"], - ["3", "43", "SomeText1", "Vial"] - ] - - - return series_query_results From de9d48d51f152d08e4109af6e19dc3537f268f4c Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Mon, 3 Feb 2025 06:52:25 -0800 Subject: [PATCH 25/59] Verify field.key exist --- specifyweb/stored_queries/execution.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index f41983b0d7e..2da09f7cbf6 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -672,8 +672,11 @@ def build_query(session, collection, user, tableid, field_specs, query = query.add_columns(formatted_field) selected_fields.append(formatted_field) - if hasattr(field, 'key') and field.key.lower() == 'catalognumber': - catalog_number_field = formatted_field + # if hasattr(field, 'key') and field.key.lower() == 'catalognumber': + # catalog_number_field = formatted_field + if hasattr(field, 'key') and field.key and field.key.lower() == 'catalognumber': + catalog_number_field = formatted_field + if sort_type is not None: order_by_exprs.append(sort_type(field)) From b40a50b017990d4200555edb0aa321ab90938ea3 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 3 Feb 2025 09:56:10 -0600 Subject: [PATCH 26/59] enforce limit on series_post_query --- specifyweb/stored_queries/execution.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 2da09f7cbf6..f88f3e74cf7 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -567,7 +567,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, query = query.limit(limit) if series: - return {'results': series_post_query(query)} + return {'results': series_post_query(query, limit)} log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -709,7 +709,7 @@ def build_query(session, collection, user, tableid, field_specs, logger.warning("query: %s", query.query) return query.query, order_by_exprs -def series_post_query(query, co_id_cat_num_pair_col_index=0): +def series_post_query(query, limit=40, co_id_cat_num_pair_col_index=0): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" @@ -745,4 +745,5 @@ def process_row(row): for co_id, cat_num_series in co_id_cat_num_consecutive_pairs ] - return [item for sublist in map(process_row, list(query)) for item in sublist] + limit = limit if limit is not None else 40 + return [item for sublist in map(process_row, list(query)) for item in sublist][:limit] From af0fc3323da4bffb163e3438dc942596992a1f31 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 3 Feb 2025 10:45:45 -0600 Subject: [PATCH 27/59] set max rows for series_post_query to return --- specifyweb/stored_queries/execution.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index f88f3e74cf7..aad47130dda 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -567,7 +567,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, query = query.limit(limit) if series: - return {'results': series_post_query(query, limit)} + return {'results': series_post_query(query)} log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -672,8 +672,6 @@ def build_query(session, collection, user, tableid, field_specs, query = query.add_columns(formatted_field) selected_fields.append(formatted_field) - # if hasattr(field, 'key') and field.key.lower() == 'catalognumber': - # catalog_number_field = formatted_field if hasattr(field, 'key') and field.key and field.key.lower() == 'catalognumber': catalog_number_field = formatted_field @@ -709,7 +707,7 @@ def build_query(session, collection, user, tableid, field_specs, logger.warning("query: %s", query.query) return query.query, order_by_exprs -def series_post_query(query, limit=40, co_id_cat_num_pair_col_index=0): +def series_post_query(query, co_id_cat_num_pair_col_index=0): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" @@ -745,5 +743,5 @@ def process_row(row): for co_id, cat_num_series in co_id_cat_num_consecutive_pairs ] - limit = limit if limit is not None else 40 - return [item for sublist in map(process_row, list(query)) for item in sublist][:limit] + MAX_ROWS = 500 + return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] From 7177b63cd2610d71b21520244d8233aed1be5b59 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 27 Feb 2025 12:23:25 -0600 Subject: [PATCH 28/59] fix series query logic to handle alphanumeric cat nums --- specifyweb/stored_queries/execution.py | 55 +++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index aad47130dda..21b5f5ca37f 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -707,7 +707,7 @@ def build_query(session, collection, user, tableid, field_specs, logger.warning("query: %s", query.query) return query.query, order_by_exprs -def series_post_query(query, co_id_cat_num_pair_col_index=0): +def series_post_query_for_int_cat_nums(query, co_id_cat_num_pair_col_index=0): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" @@ -745,3 +745,56 @@ def process_row(row): MAX_ROWS = 500 return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] + +def series_post_query(query, sort_type=0, co_id_cat_num_pair_col_index=0): + """Transform the query results by removing the co_id:catnum pair column + and adding a co_id colum and formatted catnum range column. + Sort the results by the first catnum in the range.""" + log_sqlalchemy_query(query) # Debugging + + def parse_catalog(catalog): + m = re.match(r'^([A-Za-z]*)(\d+)$', catalog) + if m: + return m.group(1), int(m.group(2)) + else: + return catalog, None + + def catalog_sort_key(x): + prefix, num = parse_catalog(x[1]) + return (prefix, num if num is not None else x[1]) + + def are_adjacent(cat1, cat2): + prefix1, num1 = parse_catalog(cat1) + prefix2, num2 = parse_catalog(cat2) + return prefix1 == prefix2 and num1 is not None and num2 is not None and (num1 + 1 == num2) + + def group_consecutive_ranges(lst): + def group_consecutives(acc, x): + if not acc or not are_adjacent(acc[-1][-1][1], x[1]): + acc.append([x]) + else: + acc[-1].append(x) + return acc + + grouped = reduce(group_consecutives, lst, []) + return [ + (','.join([x[0] for x in group]), + f"{group[0][1]} - {group[-1][1]}" if len(group) > 1 else f"{group[0][1]}") + for group in grouped + ] + + def process_row(row): + pairs = [pair.split(':') for pair in row[co_id_cat_num_pair_col_index].split(',')] + sorted_pairs = sorted(pairs, key=catalog_sort_key) + co_id_cat_num_consecutive_pairs = group_consecutive_ranges(sorted_pairs) + return [ + [co_id, cat_num_series] + ( + list(row[1:]) if co_id_cat_num_pair_col_index == 0 + else list(row[:co_id_cat_num_pair_col_index]) + list(row[co_id_cat_num_pair_col_index + 1:]) + ) + for co_id, cat_num_series in co_id_cat_num_consecutive_pairs + ] + + # Process and flatten the results + MAX_ROWS = 500 + return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] From ed0fd1e44f4e1ea6fa258e548070e7cb77dcfcb5 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 27 Feb 2025 12:26:29 -0600 Subject: [PATCH 29/59] allow for series qb records to be sorted by ranged cat nums --- specifyweb/stored_queries/execution.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 21b5f5ca37f..1febf161cdf 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -567,7 +567,12 @@ def execute(session, collection, user, tableid, distinct, series, count_only, query = query.limit(limit) if series: - return {'results': series_post_query(query)} + cat_num_sort_type = 0 + for field_spec in field_specs: + if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + cat_num_sort_type = field_spec.sort_type + break + return {'results': series_post_query(query, sort_type=cat_num_sort_type)} log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -796,5 +801,21 @@ def process_row(row): ] # Process and flatten the results + results = [item for sublist in map(process_row, list(query)) for item in sublist] + + # Reorder the final results based on sort_type + if sort_type in (1, 2): + def sort_key(record): + # record[1] contains the formatted series, e.g., "1000 - 1001" or "1003" + # Extract the first catalog number. + first_cat = record[1].split(' - ')[0].strip() + prefix, num = parse_catalog(first_cat) + return (prefix, num if num is not None else first_cat) + + reverse_order = (sort_type == 2) + results = sorted(results, key=sort_key, reverse=reverse_order) + MAX_ROWS = 500 - return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] + return results[:MAX_ROWS] + + From baf9edf65407c98acc4e4c6ad1353ee2a9152a18 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 10 Mar 2025 14:39:17 -0500 Subject: [PATCH 30/59] fix query results return limit --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 1febf161cdf..83ecf98eaf2 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -572,7 +572,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break - return {'results': series_post_query(query, sort_type=cat_num_sort_type)} + return {'results': series_post_query(query, sort_type=cat_num_sort_type)[:limit]} log_sqlalchemy_query(query) # Debugging return {'results': list(query)} From d8048de9b0c57f7304d54756c22265d268211567 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 10 Mar 2025 23:47:56 -0500 Subject: [PATCH 31/59] edit series count function --- specifyweb/stored_queries/execution.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 83ecf98eaf2..695f8856181 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -555,7 +555,10 @@ def execute(session, collection, user, tableid, distinct, series, count_only, formatauditobjs=formatauditobjs, distinct=distinct, series=series) if count_only: - return {'count': query.count()} + if series: + return {'count': len(series_post_query(query, sort_type=cat_num_sort_type))} + else: + return {'count': query.count()} else: if series: order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) @@ -572,7 +575,7 @@ def execute(session, collection, user, tableid, distinct, series, count_only, if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break - return {'results': series_post_query(query, sort_type=cat_num_sort_type)[:limit]} + return {'results': series_post_query(query, sort_type=cat_num_sort_type)} log_sqlalchemy_query(query) # Debugging return {'results': list(query)} From a4853e6ce7587eae2c21e2cc92efc7f561e08b1f Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 21 Mar 2025 12:38:39 -0500 Subject: [PATCH 32/59] handle null cat num in series --- specifyweb/specify/utils.py | 2 ++ specifyweb/stored_queries/execution.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/specifyweb/specify/utils.py b/specifyweb/specify/utils.py index ccc4b7508f8..a53cd4e6b9a 100644 --- a/specifyweb/specify/utils.py +++ b/specifyweb/specify/utils.py @@ -39,7 +39,9 @@ def log_sqlalchemy_query(query): from sqlalchemy.dialects import mysql compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) raw_sql = str(compiled_query).replace('\n', ' ') + ';' + logger.debug('='.join(['' for _ in range(80)])) logger.debug(raw_sql) + logger.debug('='.join(['' for _ in range(80)])) # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 695f8856181..3c7eb734719 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -556,6 +556,11 @@ def execute(session, collection, user, tableid, distinct, series, count_only, if count_only: if series: + cat_num_sort_type = 0 + for field_spec in field_specs: + if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + cat_num_sort_type = field_spec.sort_type + break return {'count': len(series_post_query(query, sort_type=cat_num_sort_type))} else: return {'count': query.count()} @@ -792,6 +797,9 @@ def group_consecutives(acc, x): ] def process_row(row): + if row[co_id_cat_num_pair_col_index] is None: + return [] + pairs = [pair.split(':') for pair in row[co_id_cat_num_pair_col_index].split(',')] sorted_pairs = sorted(pairs, key=catalog_sort_key) co_id_cat_num_consecutive_pairs = group_consecutive_ranges(sorted_pairs) From 1272641771bc460380608ef6623c7f8b74eff281 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 21 Mar 2025 14:49:31 -0500 Subject: [PATCH 33/59] fix merge conflict errors --- specifyweb/stored_queries/execution.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 3b4e705a932..de56be711a2 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -56,6 +56,7 @@ class BuildQueryProps(NamedTuple): replace_nulls: bool = False formatauditobjs: bool = False distinct: bool = False + series: bool = False implicit_or: bool = True format_agent_type: bool = False format_picklist: bool = False @@ -562,7 +563,7 @@ def run_ephemeral_query(collection, user, spquery): user=user, tableid=tableid, distinct=distinct, - series=series + series=series, count_only=count_only, field_specs=field_specs, limit=limit, @@ -857,7 +858,7 @@ def build_query( field_specs = [apply_specify_user_name(field_spec, user) for field_spec in field_specs] query_construct_query = None - if series and catalog_number_field: + if props.series and catalog_number_field: query_construct_query = session.query( func.group_concat( func.concat( @@ -868,10 +869,10 @@ def build_query( separator='|' ).label('co_id_catnum_paired_values') ) - elif distinct: + elif props.distinct: query_construct_query = session.query(func.group_concat(id_field.distinct(), separator=',')) else: - query_construct_query = query_construct_query = session.query(id_field) + query_construct_query = session.query(id_field) query = QueryConstruct( collection=collection, @@ -928,10 +929,10 @@ def build_query( # sort_type = SORT_TYPES[fs.sort_type] sort_type = QuerySort.by_id(fs.sort_type) - if series and fs.fieldspec.get_field().name.lower() == 'catalognumber': + if props.series and fs.fieldspec.get_field().name.lower() == 'catalognumber': continue - query, field, predicate = fs.add_to_query(query, formatauditobjs=formatauditobjs) + query, field, predicate = fs.add_to_query(query, formatauditobjs=props.formatauditobjs) if field is None: continue From 1f669f9031a43897f88a24612e9dc93e032d0ff2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 24 Mar 2025 08:48:36 -0500 Subject: [PATCH 34/59] handle int and none cat num id --- specifyweb/stored_queries/execution.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index de56be711a2..d1ae5527305 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1052,10 +1052,14 @@ def group_consecutives(acc, x): ] def process_row(row): + co_id_cat_num_seq = row[co_id_cat_num_pair_col_index] if row[co_id_cat_num_pair_col_index] is None: return [] - pairs = [pair.split(':') for pair in row[co_id_cat_num_pair_col_index].split(',')] + pre_pairs = co_id_cat_num_seq.split(',') if type(co_id_cat_num_seq) is str else None + if pre_pairs is None and type(co_id_cat_num_seq) is int: + pre_pairs = f"0:{str(co_id_cat_num_seq)}".split(',') + pairs = [pair.split(':') for pair in pre_pairs] sorted_pairs = sorted(pairs, key=catalog_sort_key) co_id_cat_num_consecutive_pairs = group_consecutive_ranges(sorted_pairs) return [ From 81e92a71a2ce6cf7f26614b3ee1975ce4be3af77 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 24 Mar 2025 16:08:39 -0500 Subject: [PATCH 35/59] execute query call fix --- specifyweb/stored_queries/batch_edit.py | 1 + specifyweb/stored_queries/views.py | 1 + 2 files changed, 2 insertions(+) diff --git a/specifyweb/stored_queries/batch_edit.py b/specifyweb/stored_queries/batch_edit.py index 98a99b7259f..ca9e9091757 100644 --- a/specifyweb/stored_queries/batch_edit.py +++ b/specifyweb/stored_queries/batch_edit.py @@ -1048,6 +1048,7 @@ def run_batch_edit_query(props: BatchEditProps): user=props["user"], tableid=tableid, distinct=True, + series=False, count_only=False, field_specs=query_with_hidden, limit=limit, diff --git a/specifyweb/stored_queries/views.py b/specifyweb/stored_queries/views.py index f368b2a342b..377b0940aa7 100644 --- a/specifyweb/stored_queries/views.py +++ b/specifyweb/stored_queries/views.py @@ -96,6 +96,7 @@ def query(request, id): user=request.specify_user, tableid=tableid, distinct=distinct, + series=series, count_only=count_only, field_specs=field_specs, limit=limit, From 728351e36a60518a9d0cd2c40fcbae4cd6331459 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 24 Mar 2025 16:09:05 -0500 Subject: [PATCH 36/59] handle series decimal cat nums --- specifyweb/stored_queries/execution.py | 105 ++++++++++++++++++------- 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index d1ae5527305..b9d0af8cb9b 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -38,6 +38,7 @@ SORT_LITERAL: Union[Literal["asc"], Literal["desc"], None] +SERIES_MAX_ROWS = 500 class QuerySort: SORT_TYPES = [None, asc, desc] @@ -794,15 +795,14 @@ def execute( return {'count': query.count()} else: if series: - order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) + # order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) # doesn't work if there are non-numeric catalog numbers + order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) logger.debug("order by: %s", order_by_exprs) query = query.order_by(*order_by_exprs).offset(offset) - if limit: - query = query.limit(limit) - if series: + query = query.limit(SERIES_MAX_ROWS) cat_num_sort_type = 0 for field_spec in field_specs: if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': @@ -810,6 +810,9 @@ def execute( break return {'results': series_post_query(query, sort_type=cat_num_sort_type)} + if limit: + query = query.limit(limit) + log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -857,7 +860,7 @@ def build_query( # field_specs = [apply_absolute_date(field_spec) for field_spec in field_specs] field_specs = [apply_specify_user_name(field_spec, user) for field_spec in field_specs] - query_construct_query = None + query_construct_query = session.query(id_field) if props.series and catalog_number_field: query_construct_query = session.query( func.group_concat( @@ -883,11 +886,7 @@ def build_query( format_agent_type=props.format_agent_type, format_picklist=props.format_picklist, ), - query=( - session.query(func.group_concat(id_field.distinct(), separator=",")) - if props.distinct - else session.query(id_field) - ), + query=query_construct_query ) tables_to_read = set( @@ -1018,23 +1017,71 @@ def series_post_query(query, sort_type=0, co_id_cat_num_pair_col_index=0): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" + log_sqlalchemy_query(query) # Debugging - def parse_catalog(catalog): + def parse_catalog_for_comparing(s): + def check_for_decimal(s): + decimal_match = re.search(r'\d+\.\d+', s) + if decimal_match: + return decimal_match.group() + return None + + try: + num = int(s) + return (num, '', '') + # return s + except ValueError: + decimal_match_str = check_for_decimal(s) + if decimal_match_str: + num, dec = decimal_match_str.split('.') + match = re.search(rf'(\D*)({num}\.{dec})(.*)', s) + if match: + prefix, number, postfix = match.groups() + prefix = prefix if prefix else '' + postfix = postfix if postfix else '' + # return (float(number), prefix, postfix) + return (int(float(number)), prefix, postfix) + + + match = re.search(r'(\D*)(\d+)(.*)', s) + if match: + prefix, number, postfix = match.groups() + prefix = prefix if prefix else '' + postfix = postfix if postfix else '' + # return (int(number), prefix, postfix) + return (int(number), prefix, postfix) + else: + return (None, s, '') + + def parse_catalog_for_sorting(catalog): m = re.match(r'^([A-Za-z]*)(\d+)$', catalog) if m: return m.group(1), int(m.group(2)) else: return catalog, None + # def catalog_sort_key(x): + # prefix, num = parse_catalog_for_sorting(x[1]) + # return (prefix, num if num is not None else x[1]) + def catalog_sort_key(x): - prefix, num = parse_catalog(x[1]) - return (prefix, num if num is not None else x[1]) + num, prefix, postfix = parse_catalog_for_comparing(x[1]) + return (prefix, num, postfix if num is not None else x[1]) + + # def are_adjacent(cat1, cat2): + # prefix1, num1 = parse_catalog(cat1) + # prefix2, num2 = parse_catalog(cat2) + # return prefix1 == prefix2 and num1 is not None and num2 is not None and (num1 + 1 == num2) def are_adjacent(cat1, cat2): - prefix1, num1 = parse_catalog(cat1) - prefix2, num2 = parse_catalog(cat2) - return prefix1 == prefix2 and num1 is not None and num2 is not None and (num1 + 1 == num2) + num1, prefix1, postfix1 = parse_catalog_for_comparing(cat1) + num2, prefix2, postfix2 = parse_catalog_for_comparing(cat2) + return prefix1 == prefix2 and \ + postfix1 == postfix2 and \ + num1 is not None and \ + num2 is not None and \ + (num1 + 1 == num2 or num1 == num2) def group_consecutive_ranges(lst): def group_consecutives(acc, x): @@ -1057,8 +1104,6 @@ def process_row(row): return [] pre_pairs = co_id_cat_num_seq.split(',') if type(co_id_cat_num_seq) is str else None - if pre_pairs is None and type(co_id_cat_num_seq) is int: - pre_pairs = f"0:{str(co_id_cat_num_seq)}".split(',') pairs = [pair.split(':') for pair in pre_pairs] sorted_pairs = sorted(pairs, key=catalog_sort_key) co_id_cat_num_consecutive_pairs = group_consecutive_ranges(sorted_pairs) @@ -1074,18 +1119,18 @@ def process_row(row): results = [item for sublist in map(process_row, list(query)) for item in sublist] # Reorder the final results based on sort_type - if sort_type in (1, 2): - def sort_key(record): - # record[1] contains the formatted series, e.g., "1000 - 1001" or "1003" - # Extract the first catalog number. - first_cat = record[1].split(' - ')[0].strip() - prefix, num = parse_catalog(first_cat) - return (prefix, num if num is not None else first_cat) + # if sort_type in (1, 2): + # def sort_key(record): + # # record[1] contains the formatted series, e.g., "1000 - 1001" or "1003" + # # Extract the first catalog number. + # first_cat = record[1].split(' - ')[0].strip() + # prefix, num = parse_catalog_for_sorting(first_cat) + # return (prefix, num if num is not None else first_cat) - reverse_order = (sort_type == 2) - results = sorted(results, key=sort_key, reverse=reverse_order) + # reverse_order = (sort_type == 2) + # results = sorted(results, key=sort_key, reverse=reverse_order) - MAX_ROWS = 500 - return results[:MAX_ROWS] + if sort_type == 2: + results = results[::-1] - + return results[:SERIES_MAX_ROWS] From 5610d16ad8ddae9df172d62a71a000a71a170642 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 00:30:51 -0500 Subject: [PATCH 37/59] better reverse series with count and offset fix --- specifyweb/stored_queries/execution.py | 88 ++++++++++++++++---------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index b9d0af8cb9b..fba2404eee0 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -16,6 +16,7 @@ from sqlalchemy.sql.expression import asc, desc, insert, literal from specifyweb.specify.field_change_info import FieldChangeInfo +from specifyweb.specify.models_by_table_id import get_table_id_by_model_name from specifyweb.stored_queries.group_concat import group_by_displayed_fields from specifyweb.specify.tree_utils import get_search_filters @@ -790,29 +791,68 @@ def execute( if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break - return {'count': len(series_post_query(query, sort_type=cat_num_sort_type))} + return {'count': len(series_post_query(query, limit=SERIES_MAX_ROWS, offset=0, sort_type=cat_num_sort_type))} else: return {'count': query.count()} else: - if series: + cat_num_col_id = None + cat_num_sort_type = None + idx = 0 + for field_spec in field_specs: + if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + cat_num_col_id = idx + cat_num_sort_type = field_spec.sort_type + break + idx += 1 + is_valid_series_query = series and \ + cat_num_col_id is not None \ + and tableid == get_table_id_by_model_name('Collectionobject') + + if is_valid_series_query: # order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) # doesn't work if there are non-numeric catalog numbers order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) logger.debug("order by: %s", order_by_exprs) query = query.order_by(*order_by_exprs).offset(offset) - if series: + if is_valid_series_query: query = query.limit(SERIES_MAX_ROWS) - cat_num_sort_type = 0 - for field_spec in field_specs: - if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': - cat_num_sort_type = field_spec.sort_type - break - return {'results': series_post_query(query, sort_type=cat_num_sort_type)} + # query = query.limit(limit) if limit else query.limit(SERIES_MAX_ROWS) + # return {'results': series_post_query(query, limit=limit, offset=offset, sort_type=cat_num_sort_type)} + return {'results': series_post_query(query, limit=SERIES_MAX_ROWS, offset=offset, sort_type=cat_num_sort_type)} if limit: query = query.limit(limit) + # else: + # cat_num_col_id = None + # idx = 0 + # for field_spec in field_specs: + # if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + # cat_num_col_id = idx + # break + # idx += 1 + # is_valid_series_query = series and cat_num_col_id and tableid == models.Collectionobject.specify_model.tableId + + # if is_valid_series_query: + # # order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) # doesn't work if there are non-numeric catalog numbers + # order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) + + # logger.debug("order by: %s", order_by_exprs) + # query = query.order_by(*order_by_exprs).offset(offset) + + # if is_valid_series_query: + # query = query.limit(limit) if limit else query.limit(SERIES_MAX_ROWS) + # cat_num_sort_type = 0 + # for field_spec in field_specs: + # if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + # cat_num_sort_type = field_spec.sort_type + # break + # return {'results': series_post_query(query, limit)} + + # if limit: + # query = query.limit(limit) + log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -1013,7 +1053,7 @@ def process_row(row): MAX_ROWS = 500 return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] -def series_post_query(query, sort_type=0, co_id_cat_num_pair_col_index=0): +def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair_col_index=0): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" @@ -1030,7 +1070,6 @@ def check_for_decimal(s): try: num = int(s) return (num, '', '') - # return s except ValueError: decimal_match_str = check_for_decimal(s) if decimal_match_str: @@ -1040,7 +1079,6 @@ def check_for_decimal(s): prefix, number, postfix = match.groups() prefix = prefix if prefix else '' postfix = postfix if postfix else '' - # return (float(number), prefix, postfix) return (int(float(number)), prefix, postfix) @@ -1049,7 +1087,6 @@ def check_for_decimal(s): prefix, number, postfix = match.groups() prefix = prefix if prefix else '' postfix = postfix if postfix else '' - # return (int(number), prefix, postfix) return (int(number), prefix, postfix) else: return (None, s, '') @@ -1061,19 +1098,10 @@ def parse_catalog_for_sorting(catalog): else: return catalog, None - # def catalog_sort_key(x): - # prefix, num = parse_catalog_for_sorting(x[1]) - # return (prefix, num if num is not None else x[1]) - def catalog_sort_key(x): num, prefix, postfix = parse_catalog_for_comparing(x[1]) return (prefix, num, postfix if num is not None else x[1]) - # def are_adjacent(cat1, cat2): - # prefix1, num1 = parse_catalog(cat1) - # prefix2, num2 = parse_catalog(cat2) - # return prefix1 == prefix2 and num1 is not None and num2 is not None and (num1 + 1 == num2) - def are_adjacent(cat1, cat2): num1, prefix1, postfix1 = parse_catalog_for_comparing(cat1) num2, prefix2, postfix2 = parse_catalog_for_comparing(cat2) @@ -1119,18 +1147,10 @@ def process_row(row): results = [item for sublist in map(process_row, list(query)) for item in sublist] # Reorder the final results based on sort_type - # if sort_type in (1, 2): - # def sort_key(record): - # # record[1] contains the formatted series, e.g., "1000 - 1001" or "1003" - # # Extract the first catalog number. - # first_cat = record[1].split(' - ')[0].strip() - # prefix, num = parse_catalog_for_sorting(first_cat) - # return (prefix, num if num is not None else first_cat) - - # reverse_order = (sort_type == 2) - # results = sorted(results, key=sort_key, reverse=reverse_order) - if sort_type == 2: results = results[::-1] - return results[:SERIES_MAX_ROWS] + series_limit = limit if limit else SERIES_MAX_ROWS + offset = offset if offset else 40 + # return results[:series_limit] + return results[offset:offset + series_limit] From ec557e91fbf626d6d589cfc47867c44c3c4ad61a Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 25 Mar 2025 08:39:01 -0700 Subject: [PATCH 38/59] Set series to false when remove cat num filter --- .../js_src/lib/components/QueryBuilder/Wrapped.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 3505c313680..40fed63b46b 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -304,6 +304,13 @@ function Wrapped({ table.name === 'CollectionObject' && state.fields.some((field) => field.mappingPath[0] === 'catalogNumber'); + React.useEffect(() => { + if (!showSeries) setQuery({ + ...query, + selectSeries: false + }) + }, [showSeries]) + return treeRanksLoaded ? ( @@ -565,8 +572,8 @@ function Wrapped({ runQuery('count')} onSubmitClick={(): void => @@ -580,13 +587,13 @@ function Wrapped({ selectDistinct: !(query.selectDistinct ?? false), }) } + onToggleHidden={setShowHiddenFields} onToggleSeries={(): void => setQuery({ ...query, selectSeries: !(query.selectSeries ?? false), }) } - onToggleHidden={setShowHiddenFields} /> {hasPermission('/querybuilder/query', 'execute') && ( From cb80b141e58a1bcba7d1f42664907e72659801d2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 11:06:46 -0500 Subject: [PATCH 39/59] SERIES_MAX_ROWS increase --- specifyweb/stored_queries/execution.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index fba2404eee0..a2d3dde1eff 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -39,7 +39,7 @@ SORT_LITERAL: Union[Literal["asc"], Literal["desc"], None] -SERIES_MAX_ROWS = 500 +SERIES_MAX_ROWS = 5000 class QuerySort: SORT_TYPES = [None, asc, desc] @@ -817,8 +817,6 @@ def execute( if is_valid_series_query: query = query.limit(SERIES_MAX_ROWS) - # query = query.limit(limit) if limit else query.limit(SERIES_MAX_ROWS) - # return {'results': series_post_query(query, limit=limit, offset=offset, sort_type=cat_num_sort_type)} return {'results': series_post_query(query, limit=SERIES_MAX_ROWS, offset=offset, sort_type=cat_num_sort_type)} if limit: From 0fb9ad68beee6e302a70ac4deb9b014b1ff60d0c Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 11:10:34 -0500 Subject: [PATCH 40/59] remove series limit --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index a2d3dde1eff..9c21f496af0 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -816,7 +816,7 @@ def execute( query = query.order_by(*order_by_exprs).offset(offset) if is_valid_series_query: - query = query.limit(SERIES_MAX_ROWS) + # query = query.limit(SERIES_MAX_ROWS) return {'results': series_post_query(query, limit=SERIES_MAX_ROWS, offset=offset, sort_type=cat_num_sort_type)} if limit: From deaeecaadf0798afd73c87fbf814ee48bc516218 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 11:40:30 -0500 Subject: [PATCH 41/59] reverse ordering fix --- specifyweb/stored_queries/execution.py | 49 +++++++------------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 9c21f496af0..e7dabcb8ed4 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -791,7 +791,7 @@ def execute( if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break - return {'count': len(series_post_query(query, limit=SERIES_MAX_ROWS, offset=0, sort_type=cat_num_sort_type))} + return {'count': len(series_post_query(query, limit=limit, offset=0, sort_type=cat_num_sort_type, is_count=True))} else: return {'count': query.count()} else: @@ -810,47 +810,22 @@ def execute( if is_valid_series_query: # order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) # doesn't work if there are non-numeric catalog numbers + # if cat_num_sort_type in {0, 1}: + # order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) + # elif cat_num_sort_type == 2: + # order_by_exprs.insert(0, text("collectionobject.`CatalogNumber` DESC")) order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) logger.debug("order by: %s", order_by_exprs) query = query.order_by(*order_by_exprs).offset(offset) - + if is_valid_series_query: # query = query.limit(SERIES_MAX_ROWS) - return {'results': series_post_query(query, limit=SERIES_MAX_ROWS, offset=offset, sort_type=cat_num_sort_type)} - + return {'results': series_post_query(query, limit=limit, offset=offset, sort_type=cat_num_sort_type)} + if limit: query = query.limit(limit) - # else: - # cat_num_col_id = None - # idx = 0 - # for field_spec in field_specs: - # if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': - # cat_num_col_id = idx - # break - # idx += 1 - # is_valid_series_query = series and cat_num_col_id and tableid == models.Collectionobject.specify_model.tableId - - # if is_valid_series_query: - # # order_by_exprs.insert(0, text("MIN(IFNULL(CAST(`CatalogNumber` AS DECIMAL(65)), NULL))")) # doesn't work if there are non-numeric catalog numbers - # order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) - - # logger.debug("order by: %s", order_by_exprs) - # query = query.order_by(*order_by_exprs).offset(offset) - - # if is_valid_series_query: - # query = query.limit(limit) if limit else query.limit(SERIES_MAX_ROWS) - # cat_num_sort_type = 0 - # for field_spec in field_specs: - # if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': - # cat_num_sort_type = field_spec.sort_type - # break - # return {'results': series_post_query(query, limit)} - - # if limit: - # query = query.limit(limit) - log_sqlalchemy_query(query) # Debugging return {'results': list(query)} @@ -1051,7 +1026,7 @@ def process_row(row): MAX_ROWS = 500 return [item for sublist in map(process_row, list(query)) for item in sublist][:MAX_ROWS] -def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair_col_index=0): +def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair_col_index=0, is_count=False): """Transform the query results by removing the co_id:catnum pair column and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" @@ -1148,7 +1123,9 @@ def process_row(row): if sort_type == 2: results = results[::-1] + if is_count: + results + series_limit = limit if limit else SERIES_MAX_ROWS - offset = offset if offset else 40 - # return results[:series_limit] + offset = offset if offset else 0 return results[offset:offset + series_limit] From 831d34bb03e9aad3f551bc0dce9a46595ed6ad75 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 11:45:40 -0500 Subject: [PATCH 42/59] limit fix --- specifyweb/stored_queries/execution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index e7dabcb8ed4..eddf6f1b982 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -39,7 +39,7 @@ SORT_LITERAL: Union[Literal["asc"], Literal["desc"], None] -SERIES_MAX_ROWS = 5000 +SERIES_MAX_ROWS = 10000 class QuerySort: SORT_TYPES = [None, asc, desc] @@ -791,7 +791,7 @@ def execute( if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break - return {'count': len(series_post_query(query, limit=limit, offset=0, sort_type=cat_num_sort_type, is_count=True))} + return {'count': len(series_post_query(query, limit=SERIES_MAX_ROWS, offset=0, sort_type=cat_num_sort_type, is_count=True))} else: return {'count': query.count()} else: From fbe3c65f695dab37f6e75e3d90d49d8ab35b80f8 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 25 Mar 2025 09:46:13 -0700 Subject: [PATCH 43/59] tests + model field --- config/specify_datamodel.xml | 1 + .../lib/components/QueryBuilder/Wrapped.tsx | 12 +- .../tests/ajax/static/context/datamodel.json | 8 + .../schema_localization.json/lang=en.json | 11 + specifyweb/specify/datamodel.py | 1 + specifyweb/specify/models.py | 1 + specifyweb/specify/temp_models.py | 1 + .../stored_queries/tests/static/co_query.json | 839 +++++++++--------- 8 files changed, 449 insertions(+), 425 deletions(-) diff --git a/config/specify_datamodel.xml b/config/specify_datamodel.xml index 84ca7855dca..f9aa1dfb940 100644 --- a/config/specify_datamodel.xml +++ b/config/specify_datamodel.xml @@ -4169,6 +4169,7 @@ + diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 40fed63b46b..bfb31409135 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -304,12 +304,12 @@ function Wrapped({ table.name === 'CollectionObject' && state.fields.some((field) => field.mappingPath[0] === 'catalogNumber'); - React.useEffect(() => { - if (!showSeries) setQuery({ - ...query, - selectSeries: false - }) - }, [showSeries]) + React.useEffect(() => { + if (!showSeries) setQuery({ + ...query, + selectSeries: false + }) + }, [showSeries]) return treeRanksLoaded ? ( diff --git a/specifyweb/frontend/js_src/lib/tests/ajax/static/context/datamodel.json b/specifyweb/frontend/js_src/lib/tests/ajax/static/context/datamodel.json index b309f727d27..93621ec6fa0 100644 --- a/specifyweb/frontend/js_src/lib/tests/ajax/static/context/datamodel.json +++ b/specifyweb/frontend/js_src/lib/tests/ajax/static/context/datamodel.json @@ -31078,6 +31078,14 @@ "required": false, "type": "java.lang.Boolean" }, + { + "name": "selectSeries", + "column": "SelectSeries", + "indexed": false, + "unique": false, + "required": false, + "type": "java.lang.Boolean" + }, { "name": "smushed", "column": "Smushed", diff --git a/specifyweb/frontend/js_src/lib/tests/ajax/static/context/schema_localization.json/lang=en.json b/specifyweb/frontend/js_src/lib/tests/ajax/static/context/schema_localization.json/lang=en.json index 3e03fe42bb0..8723da2413c 100644 --- a/specifyweb/frontend/js_src/lib/tests/ajax/static/context/schema_localization.json/lang=en.json +++ b/specifyweb/frontend/js_src/lib/tests/ajax/static/context/schema_localization.json/lang=en.json @@ -38532,6 +38532,17 @@ "name": "Select Distinct", "desc": "Select Distinct" }, + "selectseries": { + "format": null, + "ishidden": false, + "isuiformatter": false, + "picklistname": null, + "type": null, + "isrequired": false, + "weblinkname": null, + "name": "Select Series", + "desc": "Select Series" + }, "smushed": { "format": null, "ishidden": false, diff --git a/specifyweb/specify/datamodel.py b/specifyweb/specify/datamodel.py index 2c17b72b039..6382aa495fa 100644 --- a/specifyweb/specify/datamodel.py +++ b/specifyweb/specify/datamodel.py @@ -6592,6 +6592,7 @@ def is_tree_table(table: Table): Field(name='remarks', column='Remarks', indexed=False, unique=False, required=False, type='text', length=4096), Field(name='searchSynonymy', column='SearchSynonymy', indexed=False, unique=False, required=False, type='java.lang.Boolean'), Field(name='selectDistinct', column='SelectDistinct', indexed=False, unique=False, required=False, type='java.lang.Boolean'), + Field(name='selectSeries', column='SelectSeries', indexed=False, unique=False, required=False, type='java.lang.Boolean'), Field(name='smushed', column='Smushed', indexed=False, unique=False, required=False, type='java.lang.Boolean'), Field(name='sqlStr', column='SqlStr', indexed=False, unique=False, required=False, type='text', length=4096), Field(name='timestampCreated', column='TimestampCreated', indexed=False, unique=False, required=True, type='java.sql.Timestamp'), diff --git a/specifyweb/specify/models.py b/specifyweb/specify/models.py index cfcbc4e2201..14735fc7ac1 100644 --- a/specifyweb/specify/models.py +++ b/specifyweb/specify/models.py @@ -6260,6 +6260,7 @@ class Spquery(models.Model): remarks = models.TextField(blank=True, null=True, unique=False, db_column='Remarks', db_index=False) searchsynonymy = models.BooleanField(blank=True, null=True, unique=False, db_column='SearchSynonymy', db_index=False) selectdistinct = models.BooleanField(blank=True, null=True, unique=False, db_column='SelectDistinct', db_index=False) + selectseries = models.BooleanField(blank=True, null=True, unique=False, db_column='SelectSeries', db_index=False) smushed = models.BooleanField(blank=True, null=True, unique=False, db_column='Smushed', db_index=False) sqlstr = models.TextField(blank=True, null=True, unique=False, db_column='SqlStr', db_index=False) timestampcreated = models.DateTimeField(blank=False, null=False, unique=False, db_column='TimestampCreated', db_index=False, default=timezone.now) diff --git a/specifyweb/specify/temp_models.py b/specifyweb/specify/temp_models.py index c21561bf9f5..d8240c31e1e 100644 --- a/specifyweb/specify/temp_models.py +++ b/specifyweb/specify/temp_models.py @@ -5011,6 +5011,7 @@ class Spquery(models.Model): remarks = models.TextField(db_column='Remarks', blank=True, null=True) # Field name made lowercase. searchsynonymy = models.BooleanField(db_column='SearchSynonymy', blank=True, null=True) # Field name made lowercase. selectdistinct = models.BooleanField(db_column='SelectDistinct', blank=True, null=True) # Field name made lowercase. + selectseries = models.BooleanField(db_column='SelectSeries', blank=True, null=True) sqlstr = models.TextField(db_column='SqlStr', blank=True, null=True) # Field name made lowercase. modifiedbyagentid = models.ForeignKey(Agent, models.DO_NOTHING, db_column='ModifiedByAgentID', blank=True, null=True) # Field name made lowercase. specifyuserid = models.ForeignKey(Specifyuser, models.DO_NOTHING, db_column='SpecifyUserID') # Field name made lowercase. diff --git a/specifyweb/stored_queries/tests/static/co_query.json b/specifyweb/stored_queries/tests/static/co_query.json index 0d53f22fb0f..26540c8732d 100644 --- a/specifyweb/stored_queries/tests/static/co_query.json +++ b/specifyweb/stored_queries/tests/static/co_query.json @@ -1,420 +1,421 @@ { - "name": "New Query", - "contextname": "CollectionObject", - "contexttableid": 1, - "selectdistinct": false, - "countonly": true, - "formatauditrecids": false, - "specifyuser": "/api/specify/specifyuser/1/", - "isfavorite": true, - "ordinal": 32767, - "fields": [ - { - "tablelist": "1", - "stringid": "1.collectionobject.", - "fieldname": "", - "isrelfld": false, - "sorttype": 0, - "position": 0, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1", - "stringid": "1.collectionobject.catalogedDate", - "fieldname": "catalogedDate", - "isrelfld": false, - "sorttype": 0, - "position": 1, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1", - "stringid": "1.collectionobject.catalogedDateNumericDay", - "fieldname": "catalogedDateNumericDay", - "isrelfld": false, - "sorttype": 0, - "position": 2, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1", - "stringid": "1.collectionobject.catalogedDateNumericMonth", - "fieldname": "catalogedDateNumericMonth", - "isrelfld": false, - "sorttype": 0, - "position": 3, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1", - "stringid": "1.collectionobject.catalogNumber", - "fieldname": "catalogNumber", - "isrelfld": false, - "sorttype": 0, - "position": 4, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1", - "stringid": "1.collectionobject.guid", - "fieldname": "guid", - "isrelfld": false, - "sorttype": 0, - "position": 5, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.cataloger", - "fieldname": "cataloger", - "isrelfld": true, - "sorttype": 0, - "position": 6, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.abbreviation", - "fieldname": "abbreviation", - "isrelfld": false, - "sorttype": 0, - "position": 7, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.agentType", - "fieldname": "agentType", - "isrelfld": false, - "sorttype": 0, - "position": 8, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.firstName", - "fieldname": "firstName", - "isrelfld": false, - "sorttype": 0, - "position": 9, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,9-determinations", - "stringid": "1,9-determinations.determination.isCurrent", - "fieldname": "isCurrent", - "isrelfld": false, - "sorttype": 0, - "position": 10, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,9-determinations", - "stringid": "1,9-determinations.determination.determinations", - "fieldname": "determinations", - "isrelfld": true, - "sorttype": 0, - "position": 11, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,63-preparations", - "stringid": "1,63-preparations.preparation.preparations", - "fieldname": "preparations", - "isrelfld": true, - "sorttype": 0, - "position": 12, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,63-preparations", - "stringid": "1,63-preparations.preparation.text5", - "fieldname": "text5", - "isrelfld": false, - "sorttype": 0, - "position": 13, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10", - "stringid": "1,10.collectingevent.collectingEvent", - "fieldname": "collectingEvent", - "isrelfld": true, - "sorttype": 0, - "position": 14, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.locality", - "fieldname": "locality", - "isrelfld": true, - "sorttype": 0, - "position": 15, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.text2", - "fieldname": "text2", - "isrelfld": false, - "sorttype": 0, - "position": 16, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.remarks", - "fieldname": "remarks", - "isrelfld": false, - "sorttype": 0, - "position": 17, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2,3", - "stringid": "1,10,2,3.geography.Continent", - "fieldname": "Continent", - "isrelfld": false, - "sorttype": 0, - "position": 18, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2,3", - "stringid": "1,10,2,3.geography.geographyCode", - "fieldname": "geographyCode", - "isrelfld": false, - "sorttype": 0, - "position": 19, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2,3", - "stringid": "1,10,2,3.geography.Country", - "fieldname": "Country", - "isrelfld": false, - "sorttype": 0, - "position": 20, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2,3", - "stringid": "1,10,2,3.geography.Province", - "fieldname": "Province", - "isrelfld": false, - "sorttype": 0, - "position": 21, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2,3", - "stringid": "1,10,2,3.geography.County", - "fieldname": "County", - "isrelfld": false, - "sorttype": 0, - "position": 22, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,9-determinations,4", - "stringid": "1,9-determinations,4.taxon.Subspecies", - "fieldname": "Subspecies", - "isrelfld": false, - "sorttype": 0, - "position": 23, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,9-determinations,4", - "stringid": "1,9-determinations,4.taxon.Species", - "fieldname": "Species", - "isrelfld": false, - "sorttype": 0, - "position": 24, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,9-determinations,4", - "stringid": "1,9-determinations,4.taxon.Genus", - "fieldname": "Genus", - "isrelfld": false, - "sorttype": 0, - "position": 25, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.latitude1", - "fieldname": "latitude1", - "isrelfld": false, - "sorttype": 0, - "position": 26, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.longitude1", - "fieldname": "longitude1", - "isrelfld": false, - "sorttype": 0, - "position": 27, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.latitude2", - "fieldname": "latitude2", - "isrelfld": false, - "sorttype": 0, - "position": 28, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.longitude2", - "fieldname": "longitude2", - "isrelfld": false, - "sorttype": 0, - "position": 29, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.latLongType", - "fieldname": "latLongType", - "isrelfld": false, - "sorttype": 0, - "position": 30, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.latLongAccuracy", - "fieldname": "latLongAccuracy", - "isrelfld": false, - "sorttype": 0, - "position": 31, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - }, - { - "tablelist": "1,10,2", - "stringid": "1,10,2.locality.localityId", - "fieldname": "localityId", - "isrelfld": false, - "sorttype": 0, - "position": 32, - "isdisplay": true, - "operstart": 8, - "startvalue": "", - "isnot": false - } - ], - "_tablename": "SpQuery", - "remarks": null, - "searchsynonymy": null, - "smushed": null, - "sqlstr": null, - "timestampcreated": "2024-08-20", - "timestampmodified": null, - "version": 1, - "createdbyagent": null, - "modifiedbyagent": null, - "limit": 40 -} \ No newline at end of file + "name": "New Query", + "contextname": "CollectionObject", + "contexttableid": 1, + "selectdistinct": false, + "selectseries": false, + "countonly": true, + "formatauditrecids": false, + "specifyuser": "/api/specify/specifyuser/1/", + "isfavorite": true, + "ordinal": 32767, + "fields": [ + { + "tablelist": "1", + "stringid": "1.collectionobject.", + "fieldname": "", + "isrelfld": false, + "sorttype": 0, + "position": 0, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1", + "stringid": "1.collectionobject.catalogedDate", + "fieldname": "catalogedDate", + "isrelfld": false, + "sorttype": 0, + "position": 1, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1", + "stringid": "1.collectionobject.catalogedDateNumericDay", + "fieldname": "catalogedDateNumericDay", + "isrelfld": false, + "sorttype": 0, + "position": 2, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1", + "stringid": "1.collectionobject.catalogedDateNumericMonth", + "fieldname": "catalogedDateNumericMonth", + "isrelfld": false, + "sorttype": 0, + "position": 3, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1", + "stringid": "1.collectionobject.catalogNumber", + "fieldname": "catalogNumber", + "isrelfld": false, + "sorttype": 0, + "position": 4, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1", + "stringid": "1.collectionobject.guid", + "fieldname": "guid", + "isrelfld": false, + "sorttype": 0, + "position": 5, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.cataloger", + "fieldname": "cataloger", + "isrelfld": true, + "sorttype": 0, + "position": 6, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.abbreviation", + "fieldname": "abbreviation", + "isrelfld": false, + "sorttype": 0, + "position": 7, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.agentType", + "fieldname": "agentType", + "isrelfld": false, + "sorttype": 0, + "position": 8, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.firstName", + "fieldname": "firstName", + "isrelfld": false, + "sorttype": 0, + "position": 9, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,9-determinations", + "stringid": "1,9-determinations.determination.isCurrent", + "fieldname": "isCurrent", + "isrelfld": false, + "sorttype": 0, + "position": 10, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,9-determinations", + "stringid": "1,9-determinations.determination.determinations", + "fieldname": "determinations", + "isrelfld": true, + "sorttype": 0, + "position": 11, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,63-preparations", + "stringid": "1,63-preparations.preparation.preparations", + "fieldname": "preparations", + "isrelfld": true, + "sorttype": 0, + "position": 12, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,63-preparations", + "stringid": "1,63-preparations.preparation.text5", + "fieldname": "text5", + "isrelfld": false, + "sorttype": 0, + "position": 13, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10", + "stringid": "1,10.collectingevent.collectingEvent", + "fieldname": "collectingEvent", + "isrelfld": true, + "sorttype": 0, + "position": 14, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.locality", + "fieldname": "locality", + "isrelfld": true, + "sorttype": 0, + "position": 15, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.text2", + "fieldname": "text2", + "isrelfld": false, + "sorttype": 0, + "position": 16, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.remarks", + "fieldname": "remarks", + "isrelfld": false, + "sorttype": 0, + "position": 17, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2,3", + "stringid": "1,10,2,3.geography.Continent", + "fieldname": "Continent", + "isrelfld": false, + "sorttype": 0, + "position": 18, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2,3", + "stringid": "1,10,2,3.geography.geographyCode", + "fieldname": "geographyCode", + "isrelfld": false, + "sorttype": 0, + "position": 19, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2,3", + "stringid": "1,10,2,3.geography.Country", + "fieldname": "Country", + "isrelfld": false, + "sorttype": 0, + "position": 20, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2,3", + "stringid": "1,10,2,3.geography.Province", + "fieldname": "Province", + "isrelfld": false, + "sorttype": 0, + "position": 21, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2,3", + "stringid": "1,10,2,3.geography.County", + "fieldname": "County", + "isrelfld": false, + "sorttype": 0, + "position": 22, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,9-determinations,4", + "stringid": "1,9-determinations,4.taxon.Subspecies", + "fieldname": "Subspecies", + "isrelfld": false, + "sorttype": 0, + "position": 23, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,9-determinations,4", + "stringid": "1,9-determinations,4.taxon.Species", + "fieldname": "Species", + "isrelfld": false, + "sorttype": 0, + "position": 24, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,9-determinations,4", + "stringid": "1,9-determinations,4.taxon.Genus", + "fieldname": "Genus", + "isrelfld": false, + "sorttype": 0, + "position": 25, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.latitude1", + "fieldname": "latitude1", + "isrelfld": false, + "sorttype": 0, + "position": 26, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.longitude1", + "fieldname": "longitude1", + "isrelfld": false, + "sorttype": 0, + "position": 27, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.latitude2", + "fieldname": "latitude2", + "isrelfld": false, + "sorttype": 0, + "position": 28, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.longitude2", + "fieldname": "longitude2", + "isrelfld": false, + "sorttype": 0, + "position": 29, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.latLongType", + "fieldname": "latLongType", + "isrelfld": false, + "sorttype": 0, + "position": 30, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.latLongAccuracy", + "fieldname": "latLongAccuracy", + "isrelfld": false, + "sorttype": 0, + "position": 31, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + }, + { + "tablelist": "1,10,2", + "stringid": "1,10,2.locality.localityId", + "fieldname": "localityId", + "isrelfld": false, + "sorttype": 0, + "position": 32, + "isdisplay": true, + "operstart": 8, + "startvalue": "", + "isnot": false + } + ], + "_tablename": "SpQuery", + "remarks": null, + "searchsynonymy": null, + "smushed": null, + "sqlstr": null, + "timestampcreated": "2024-08-20", + "timestampmodified": null, + "version": 1, + "createdbyagent": null, + "modifiedbyagent": null, + "limit": 40 +} From 8c136edcf0ef608b4d9f4f4b06396850564cace9 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 11:58:53 -0500 Subject: [PATCH 44/59] offset fix --- specifyweb/stored_queries/execution.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index eddf6f1b982..c1f413839a9 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -815,14 +815,13 @@ def execute( # elif cat_num_sort_type == 2: # order_by_exprs.insert(0, text("collectionobject.`CatalogNumber` DESC")) order_by_exprs.insert(0, text("collectionobject.`CatalogNumber`")) - - logger.debug("order by: %s", order_by_exprs) - query = query.order_by(*order_by_exprs).offset(offset) - if is_valid_series_query: # query = query.limit(SERIES_MAX_ROWS) return {'results': series_post_query(query, limit=limit, offset=offset, sort_type=cat_num_sort_type)} + logger.debug("order by: %s", order_by_exprs) + query = query.order_by(*order_by_exprs).offset(offset) + if limit: query = query.limit(limit) @@ -1124,7 +1123,7 @@ def process_row(row): results = results[::-1] if is_count: - results + return results series_limit = limit if limit else SERIES_MAX_ROWS offset = offset if offset else 0 From e658030941f6b5dbe2bcc9f1f5986b4f564e38f2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 25 Mar 2025 12:06:23 -0500 Subject: [PATCH 45/59] selectseries django migration --- .../specify/migrations/0027_selectseries.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 specifyweb/specify/migrations/0027_selectseries.py diff --git a/specifyweb/specify/migrations/0027_selectseries.py b/specifyweb/specify/migrations/0027_selectseries.py new file mode 100644 index 00000000000..dae38a3120d --- /dev/null +++ b/specifyweb/specify/migrations/0027_selectseries.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2025-03-25 17:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('specify', '0026_taxontreedef_alter_delete'), + ] + + operations = [ + migrations.AddField( + model_name='spquery', + name='selectseries', + field=models.BooleanField(blank=True, db_column='SelectSeries', null=True), + ), + ] From 9f7125f92e9996d6a0ef771e3aaf65b519815824 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 11 Apr 2025 09:52:23 -0500 Subject: [PATCH 46/59] fix predicates on catanum field in series queries --- specifyweb/stored_queries/execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index c1f413839a9..843cb3c1317 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -941,6 +941,8 @@ def build_query( sort_type = QuerySort.by_id(fs.sort_type) if props.series and fs.fieldspec.get_field().name.lower() == 'catalognumber': + _, _, predicate = fs.add_to_query(query, formatauditobjs=props.formatauditobjs) + predicates_by_field[fs.fieldspec].append(predicate) if predicate is not None else None continue query, field, predicate = fs.add_to_query(query, formatauditobjs=props.formatauditobjs) From 7aceebaaf5141b884bccd2508538321e08982c99 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 11 Apr 2025 15:28:06 +0000 Subject: [PATCH 47/59] Lint code with ESLint and Prettier Triggered by c56933fac878130442a4ba88dfb440578479ab2e on branch refs/heads/issue-2000 --- .../FormSliders/IntegratedRecordSelector.tsx | 482 +++++++++--------- .../components/Interactions/PrepDialog.tsx | 3 +- .../components/Interactions/PrepDialogRow.tsx | 6 +- .../lib/components/QueryBuilder/Wrapped.tsx | 11 +- .../js_src/lib/components/Reports/index.tsx | 6 +- .../lib/components/SearchDialog/index.tsx | 18 +- .../js_src/lib/localization/attachments.ts | 6 +- .../js_src/lib/localization/interactions.ts | 4 +- 8 files changed, 272 insertions(+), 264 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/FormSliders/IntegratedRecordSelector.tsx b/specifyweb/frontend/js_src/lib/components/FormSliders/IntegratedRecordSelector.tsx index 2970054414e..08a0e65e2f7 100644 --- a/specifyweb/frontend/js_src/lib/components/FormSliders/IntegratedRecordSelector.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormSliders/IntegratedRecordSelector.tsx @@ -169,261 +169,263 @@ export function IntegratedRecordSelector({ isLoanPrep && (collection.related?.isNew() === true || collection.related?.needsSaved); - const [isWarningOpen, handleWarning,closeWarning] = useBooleanState() + const [isWarningOpen, handleWarning, closeWarning] = useBooleanState(); - return ( - - { - if (isInteraction) { - setInteractionResource(resources[0]); - handleOpenDialog(); - } - if (!isInteraction && formType !== 'formTable') - collection.add(resources); - handleAdding(resources); - }} - onDelete={(...args): void => { - if (isCollapsed) handleExpand(); - handleDelete?.(...args); - }} - onFetch={handleFetch} - onSlide={(index): void => { - handleExpand(); - if (typeof urlParameter === 'string') setIndex(index.toString()); - }} - {...rest} - > - {({ - dialogs, - slider, - resource, - onAdd: handleAdd, - onRemove: handleRemove, - showSearchDialog, - isLoading, - }): JSX.Element => ( - <> - {isInteraction && - typeof collection.related === 'object' && - isDialogOpen ? ( - - } - interactionResource={interactionResource} - itemCollection={ - collection as Collection - } - onClose={handleCloseDialog} - /> - ) : undefined} - {formType === 'form' ? ( - - ( - <> - + { + if (isInteraction) { + setInteractionResource(resources[0]); + handleOpenDialog(); + } + if (!isInteraction && formType !== 'formTable') + collection.add(resources); + handleAdding(resources); + }} + onDelete={(...args): void => { + if (isCollapsed) handleExpand(); + handleDelete?.(...args); + }} + onFetch={handleFetch} + onSlide={(index): void => { + handleExpand(); + if (typeof urlParameter === 'string') setIndex(index.toString()); + }} + {...rest} + > + {({ + dialogs, + slider, + resource, + onAdd: handleAdd, + onRemove: handleRemove, + showSearchDialog, + isLoading, + }): JSX.Element => ( + <> + {isInteraction && + typeof collection.related === 'object' && + isDialogOpen ? ( + + } + interactionResource={interactionResource} + itemCollection={ + collection as Collection + } + onClose={handleCloseDialog} + /> + ) : undefined} + {formType === 'form' ? ( + + ( + <> + + {!isDependent && + hasTablePermission( + relationship.relatedTable.name, + 'read' + ) && + typeof handleAdd === 'function' ? ( + 0) || + isTaxonTreeDefItemTable } + onClick={showSearchDialog} /> - {!isDependent && - hasTablePermission( - relationship.relatedTable.name, - 'read' - ) && - typeof handleAdd === 'function' ? ( - 0) || - isTaxonTreeDefItemTable + ) : undefined} + {hasTablePermission( + relationship.relatedTable.name, + 'create' + ) && typeof handleAdd === 'function' ? ( + isCOJO ? ( + } - onClick={showSearchDialog} /> - ) : undefined} - {hasTablePermission( - relationship.relatedTable.name, - 'create' - ) && typeof handleAdd === 'function' ? ( - isCOJO ? ( - - } - /> - ) : ( - 0) || - isTaxonTreeDefItemTable - } - onClick={(): void => { - const resource = - new collection.table.specifyTable.Resource(); - - if ( - isDependent || - viewName === relationship.relatedTable.view - ) { - focusFirstField(); - handleAdd([resource]); - return; - } - - if (state.type === 'AddResourceState') - setState({ type: 'MainState' }); - else - setState({ - type: 'AddResourceState', - resource, - handleAdd, - }); - }} - /> - ) - ) : undefined} - {hasTablePermission( - relationship.relatedTable.name, - isDependent ? 'delete' : 'read' - ) && typeof handleRemove === 'function' ? ( - 0) || + isTaxonTreeDefItemTable } onClick={(): void => { - if (isAttachmentTable) { - handleWarning() - } else { - handleRemove('minusButton'); + const resource = + new collection.table.specifyTable.Resource(); + + if ( + isDependent || + viewName === relationship.relatedTable.view + ) { + focusFirstField(); + handleAdd([resource]); + return; } + + if (state.type === 'AddResourceState') + setState({ type: 'MainState' }); + else + setState({ + type: 'AddResourceState', + resource, + handleAdd, + }); }} /> - ) : undefined} - { + if (isAttachmentTable) { + handleWarning(); + } else { + handleRemove('minusButton'); + } + }} /> - {isAttachmentTable && ( - - )} - {specifyNetworkBadge} - {!isToOne && slider} - - )} - isCollapsed={isCollapsed} - isDependent={isDependent} - isLoading={isLoading} - isSubForm={dialog === false} - key={resource?.cid} - preHeaderButtons={collapsibleButton} - resource={resource} - title={relationship.label} - onAdd={undefined} - onDeleted={ - collection.models.length <= 1 ? handleClose : undefined - } - onSaved={handleClose} - viewName={viewName} - /* - * Don't save the resource on save button click if it is a dependent - * resource - */ - onClose={handleClose} - /> - - ) : null} - {formType === 'formTable' ? ( - + {isAttachmentTable && ( + + )} + {specifyNetworkBadge} + {!isToOne && slider} + + )} isCollapsed={isCollapsed} + isDependent={isDependent} + isLoading={isLoading} + isSubForm={dialog === false} + key={resource?.cid} preHeaderButtons={collapsibleButton} - sortField={sortField} - viewName={viewName} - onAdd={ - isTaxonTreeDefItemTable - ? undefined - : (resources): void => { - if (!isInteraction) collection.add(resources); - handleAdd?.(resources); - } + resource={resource} + title={relationship.label} + onAdd={undefined} + onDeleted={ + collection.models.length <= 1 ? handleClose : undefined } + onSaved={handleClose} + viewName={viewName} + /* + * Don't save the resource on save button click if it is a dependent + * resource + */ onClose={handleClose} - onDelete={(resource, index): void => { - if (isAttachmentTable) { - handleWarning() - } else { - collection.remove(resource); - if (isCollapsed) handleExpand(); - handleDelete?.(index, 'minusButton'); - } - }} - onFetchMore={handleFetch} - /> - ) : null} - {dialogs} - {state.type === 'AddResourceState' && - typeof handleAdd === 'function' ? ( - setState({ type: 'MainState' })} - onDeleted={undefined} - onSaved={(): void => { - state.handleAdd([state.resource]); - setState({ type: 'MainState' }); - }} /> - ) : null} - {isWarningOpen && typeof handleRemove === 'function' && isAttachmentTable? - - : undefined - } - - )} - - - ); - } + + ) : null} + {formType === 'formTable' ? ( + { + if (!isInteraction) collection.add(resources); + handleAdd?.(resources); + } + } + onClose={handleClose} + onDelete={(resource, index): void => { + if (isAttachmentTable) { + handleWarning(); + } else { + collection.remove(resource); + if (isCollapsed) handleExpand(); + handleDelete?.(index, 'minusButton'); + } + }} + onFetchMore={handleFetch} + /> + ) : null} + {dialogs} + {state.type === 'AddResourceState' && + typeof handleAdd === 'function' ? ( + setState({ type: 'MainState' })} + onDeleted={undefined} + onSaved={(): void => { + state.handleAdd([state.resource]); + setState({ type: 'MainState' }); + }} + /> + ) : null} + {isWarningOpen && + typeof handleRemove === 'function' && + isAttachmentTable ? ( + + ) : undefined} + + )} + + + ); +} diff --git a/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialog.tsx b/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialog.tsx index e2428ab6ff5..54b60311193 100644 --- a/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialog.tsx +++ b/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialog.tsx @@ -176,7 +176,8 @@ export function PrepDialog({ const items = filterArray( preparations.map((preparation, index) => { - if (selected[index] === 0 || Number.isNaN(selected[index])) return undefined; + if (selected[index] === 0 || Number.isNaN(selected[index])) + return undefined; const result = new itemTable.Resource(); result.set( diff --git a/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialogRow.tsx b/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialogRow.tsx index d9cb47f06c8..32ea7169b79 100644 --- a/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialogRow.tsx +++ b/specifyweb/frontend/js_src/lib/components/Interactions/PrepDialogRow.tsx @@ -82,11 +82,13 @@ export function PrepDialogRow({ {preparation.taxon ? ( - + {localized(preparation.taxon)} ) : ( - {interactionsText.notAvailable()} + {interactionsText.notAvailable()} )} {preparation.prepType} diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 4c92f964a87..6b8f8249138 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -306,11 +306,12 @@ function Wrapped({ state.fields.some((field) => field.mappingPath[0] === 'catalogNumber'); React.useEffect(() => { - if (!showSeries) setQuery({ - ...query, - selectSeries: false - }) - }, [showSeries]) + if (!showSeries) + setQuery({ + ...query, + selectSeries: false, + }); + }, [showSeries]); return treeRanksLoaded ? ( diff --git a/specifyweb/frontend/js_src/lib/components/Reports/index.tsx b/specifyweb/frontend/js_src/lib/components/Reports/index.tsx index a2e6d3c879b..971a3557b20 100644 --- a/specifyweb/frontend/js_src/lib/components/Reports/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/Reports/index.tsx @@ -248,9 +248,9 @@ function ReportRow({ - + diff --git a/specifyweb/frontend/js_src/lib/components/SearchDialog/index.tsx b/specifyweb/frontend/js_src/lib/components/SearchDialog/index.tsx index f0c77508b93..0b7ae8fb7f9 100644 --- a/specifyweb/frontend/js_src/lib/components/SearchDialog/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/SearchDialog/index.tsx @@ -93,7 +93,7 @@ export function SearchDialog( }} /> ) : ( - + ); } @@ -186,7 +186,7 @@ function SearchForm({ onClose: handleClose, onUseQueryBuilder: handleUseQueryBuilder, onAdd: handleAdd, - multiple + multiple, }: { readonly forceCollection: number | undefined; readonly extraFilters: RA> | undefined; @@ -198,7 +198,7 @@ function SearchForm({ readonly onAdd?: | ((resources: RA>) => void) | undefined; - readonly multiple?: boolean + readonly multiple?: boolean; }): JSX.Element | null { const templateResource = React.useMemo( () => @@ -242,11 +242,13 @@ function SearchForm({ {queryText.queryBuilder()} - {multiple === true && } + {multiple === true && ( + + )} {commonText.search()} diff --git a/specifyweb/frontend/js_src/lib/localization/attachments.ts b/specifyweb/frontend/js_src/lib/localization/attachments.ts index ea0db3bde7b..d99287b0699 100644 --- a/specifyweb/frontend/js_src/lib/localization/attachments.ts +++ b/specifyweb/frontend/js_src/lib/localization/attachments.ts @@ -683,9 +683,9 @@ export const attachmentsText = createDictionary({ 'en-us': 'Download all found attachments', }, deleteAttachmentWarning: { - 'en-us': 'Are you sure you want to delete this attachment?' + 'en-us': 'Are you sure you want to delete this attachment?', }, attachmentDelition: { - 'en-us': 'Attachment deletion' - } + 'en-us': 'Attachment deletion', + }, } as const); diff --git a/specifyweb/frontend/js_src/lib/localization/interactions.ts b/specifyweb/frontend/js_src/lib/localization/interactions.ts index 34160d9b5c0..83d75f4c79e 100644 --- a/specifyweb/frontend/js_src/lib/localization/interactions.ts +++ b/specifyweb/frontend/js_src/lib/localization/interactions.ts @@ -388,6 +388,6 @@ export const interactionsText = createDictionary({ 'uk-ua': 'Продовжуйте без підготовки', }, notAvailable: { - 'en-us': 'Not available' - } + 'en-us': 'Not available', + }, } as const); From 285de33546bf3901ce0f48993d7444e229a815b2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 15 Apr 2025 02:10:16 -0500 Subject: [PATCH 48/59] add parsing of more catnum formats for series grouping logic --- specifyweb/stored_queries/execution.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 843cb3c1317..40524d0b09b 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1062,8 +1062,23 @@ def check_for_decimal(s): prefix = prefix if prefix else '' postfix = postfix if postfix else '' return (int(number), prefix, postfix) - else: - return (None, s, '') + + # Match integer-integer string, like "1234-5678" so that the number 12345678 is parsed + match = re.search(r'(\d+)(\D*)(\d+)', s) + if match: + prefix, num1, num2 = match.groups() + prefix = prefix if prefix else '' + combined_number = int(str(num1) + str(num2)) + return (combined_number, prefix, '') + + # Match string-interger string, like "abc-1234" so that the number 1234 is parsed + match = re.search(r'(\D*)(\d+)', s) + if match: + prefix, number = match.groups() + prefix = prefix if prefix else '' + return (int(number), prefix, '') + + return (None, s, '') def parse_catalog_for_sorting(catalog): m = re.match(r'^([A-Za-z]*)(\d+)$', catalog) From d54a6c825af26ac6c14f289dffd43546ee596d29 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 15 Apr 2025 09:28:00 -0500 Subject: [PATCH 49/59] reorder format parsing --- specifyweb/stored_queries/execution.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 40524d0b09b..bd96f672f49 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1054,14 +1054,6 @@ def check_for_decimal(s): prefix = prefix if prefix else '' postfix = postfix if postfix else '' return (int(float(number)), prefix, postfix) - - - match = re.search(r'(\D*)(\d+)(.*)', s) - if match: - prefix, number, postfix = match.groups() - prefix = prefix if prefix else '' - postfix = postfix if postfix else '' - return (int(number), prefix, postfix) # Match integer-integer string, like "1234-5678" so that the number 12345678 is parsed match = re.search(r'(\d+)(\D*)(\d+)', s) @@ -1078,6 +1070,13 @@ def check_for_decimal(s): prefix = prefix if prefix else '' return (int(number), prefix, '') + match = re.search(r'(\D*)(\d+)(.*)', s) + if match: + prefix, number, postfix = match.groups() + prefix = prefix if prefix else '' + postfix = postfix if postfix else '' + return (int(number), prefix, postfix) + return (None, s, '') def parse_catalog_for_sorting(catalog): From 595bea6b0c21a7bf27fd76a9b740a094d07eb568 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 15 Apr 2025 09:48:05 -0500 Subject: [PATCH 50/59] match correction --- specifyweb/stored_queries/execution.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index bd96f672f49..d0544a8e1af 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1056,12 +1056,11 @@ def check_for_decimal(s): return (int(float(number)), prefix, postfix) # Match integer-integer string, like "1234-5678" so that the number 12345678 is parsed - match = re.search(r'(\d+)(\D*)(\d+)', s) + match = re.search(r'^(\d+)-(\d+)$', s) if match: - prefix, num1, num2 = match.groups() - prefix = prefix if prefix else '' - combined_number = int(str(num1) + str(num2)) - return (combined_number, prefix, '') + num1, num2 = match.groups() + combined_number = int(str(num1) + str(num2)) # Concatenate as strings, then convert to int + return (combined_number, '', '') # Match string-interger string, like "abc-1234" so that the number 1234 is parsed match = re.search(r'(\D*)(\d+)', s) From c00f2a1d38bccec62852558ac81644f2a7f3d18d Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 17 Apr 2025 11:14:53 -0500 Subject: [PATCH 51/59] co formatted query error fix --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index d0544a8e1af..e6084795486 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -799,7 +799,7 @@ def execute( cat_num_sort_type = None idx = 0 for field_spec in field_specs: - if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + if field_spec.fieldspec.get_field() and field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_col_id = idx cat_num_sort_type = field_spec.sort_type break From 762e9e2dadf454588499df5602a465097070d7ad Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 18 Apr 2025 11:00:50 -0500 Subject: [PATCH 52/59] cog_inheritance_post_query_processing fix --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 23e13f2df5b..ef742fbca14 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1168,7 +1168,7 @@ def process_row(row): return results[offset:offset + series_limit] def cog_inheritance_post_query_processing(query, tableid, field_specs, collection, user): - if tableid == 1 and 'catalogNumber' in [fs.fieldspec.join_path[0].name for fs in field_specs]: + if tableid == 1 and 'catalogNumber' in [fs.fieldspec.join_path[0].name for fs in field_specs if fs.fieldspec.join_path]: if not get_cat_num_inheritance_setting(collection, user): # query = query.filter(collectionobjectgroupjoin_1.isprimary == 1) return list(query) From 4c4759585514ea84abf8e35ce64a091cb86a80b4 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 18 Apr 2025 11:52:18 -0500 Subject: [PATCH 53/59] other join_path error fix --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index ef742fbca14..ce1088dd13f 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1174,7 +1174,7 @@ def cog_inheritance_post_query_processing(query, tableid, field_specs, collectio return list(query) # Get the catalogNumber field index - catalog_number_field_index = [fs.fieldspec.join_path[0].name for fs in field_specs].index('catalogNumber') + 1 + catalog_number_field_index = [fs.fieldspec.join_path[0].name for fs in field_specs if fs.fieldspec.join_path].index('catalogNumber') + 1 if field_specs[catalog_number_field_index - 1].op_num != 1: return list(query) From 0f78de311d22ae3012af5c1cb8945e38eca60a8f Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 18 Apr 2025 14:11:22 -0500 Subject: [PATCH 54/59] null get field fix --- specifyweb/stored_queries/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index ce1088dd13f..85c79de469d 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -809,7 +809,7 @@ def execute( if series: cat_num_sort_type = 0 for field_spec in field_specs: - if field_spec.fieldspec.get_field().name.lower() == 'catalognumber': + if field_spec.fieldspec.get_field() and field_spec.fieldspec.get_field().name.lower() == 'catalognumber': cat_num_sort_type = field_spec.sort_type break return {'count': len(series_post_query(query, limit=SERIES_MAX_ROWS, offset=0, sort_type=cat_num_sort_type, is_count=True))} From cf65548df7d535002f9b91903fccce3ce9c1cb00 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 29 Apr 2025 10:38:33 -0500 Subject: [PATCH 55/59] Update and rename 0027_selectseries.py to 0028_selectseries.py --- .../migrations/{0027_selectseries.py => 0028_selectseries.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename specifyweb/specify/migrations/{0027_selectseries.py => 0028_selectseries.py} (87%) diff --git a/specifyweb/specify/migrations/0027_selectseries.py b/specifyweb/specify/migrations/0028_selectseries.py similarity index 87% rename from specifyweb/specify/migrations/0027_selectseries.py rename to specifyweb/specify/migrations/0028_selectseries.py index dae38a3120d..0769422db0b 100644 --- a/specifyweb/specify/migrations/0027_selectseries.py +++ b/specifyweb/specify/migrations/0028_selectseries.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('specify', '0026_taxontreedef_alter_delete'), + ('specify', '0027_CO_children'), ] operations = [ From d2d0794bdc7556b2d3011649ff7151af945018ba Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 6 May 2025 08:17:52 -0400 Subject: [PATCH 56/59] Hide serie checkbox if cat num field hidden --- .../frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 6b8f8249138..9015546c443 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -303,7 +303,9 @@ function Wrapped({ const showSeries = table.name === 'CollectionObject' && - state.fields.some((field) => field.mappingPath[0] === 'catalogNumber'); + state.fields.some( + (field) => field.mappingPath[0] === 'catalogNumber' && field.isDisplay + ); React.useEffect(() => { if (!showSeries) From fdce581e684375851e061da6ccf8cac2eed35d97 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 6 May 2025 08:22:51 -0400 Subject: [PATCH 57/59] Update show series when fields update --- .../js_src/lib/components/QueryBuilder/Wrapped.tsx | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx index 9015546c443..3b25a76424f 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryBuilder/Wrapped.tsx @@ -301,11 +301,14 @@ function Wrapped({ undefined ); - const showSeries = - table.name === 'CollectionObject' && - state.fields.some( - (field) => field.mappingPath[0] === 'catalogNumber' && field.isDisplay - ); + const showSeries = React.useMemo( + () => + table.name === 'CollectionObject' && + state.fields.some( + (field) => field.mappingPath[0] === 'catalogNumber' && field.isDisplay + ), + [state, table.name] + ); React.useEffect(() => { if (!showSeries) From f35c3ee940db185b5f9c03a4e9686eec2a949420 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 6 May 2025 11:24:46 -0500 Subject: [PATCH 58/59] formatted_field fix --- specifyweb/stored_queries/execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index d260053cbc6..5622de86173 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -975,6 +975,7 @@ def build_query( if field is None: continue + formatted_field = None if fs.display: formatted_field = query.objectformatter.fieldformat(fs, field) query = query.add_columns(formatted_field) From e8b8dd4ec4ac7ce7ab09d9e69d9e59dbf08fbfa9 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 8 May 2025 09:35:46 -0500 Subject: [PATCH 59/59] fix sorted_pairs in series qb --- specifyweb/stored_queries/execution.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specifyweb/stored_queries/execution.py b/specifyweb/stored_queries/execution.py index 5622de86173..dbfb6b181b8 100644 --- a/specifyweb/stored_queries/execution.py +++ b/specifyweb/stored_queries/execution.py @@ -1144,9 +1144,12 @@ def process_row(row): if row[co_id_cat_num_pair_col_index] is None: return [] - pre_pairs = co_id_cat_num_seq.split(',') if type(co_id_cat_num_seq) is str else None - pairs = [pair.split(':') for pair in pre_pairs] - sorted_pairs = sorted(pairs, key=catalog_sort_key) + sorted_pairs = sorted( + [[item if item else '0' + for item in pair.split(':')] + for pair in co_id_cat_num_seq.split(',') if isinstance(co_id_cat_num_seq, str)], + key=catalog_sort_key + ) co_id_cat_num_consecutive_pairs = group_consecutive_ranges(sorted_pairs) return [ [co_id, cat_num_series] + (