diff --git a/packages/app/app/initializers/config.js b/packages/app/app/initializers/config.js index cf3b3701d..e32333bb6 100644 --- a/packages/app/app/initializers/config.js +++ b/packages/app/app/initializers/config.js @@ -14,5 +14,5 @@ export function initialize() { export default { name: 'config', - initialize + initialize, }; diff --git a/packages/app/config/environment.js b/packages/app/config/environment.js index dd9437cf3..e24a70fae 100644 --- a/packages/app/config/environment.js +++ b/packages/app/config/environment.js @@ -39,7 +39,11 @@ module.exports = function (environment) { }; if (environment === 'development') { - ENV['ember-cli-mirage'] = { enabled: !(process.env.DISABLE_MOCKS || process.env.APP_ENV === 'localElide') }; + ENV['ember-cli-mirage'] = { + enabled: !( + process.env.DISABLE_MOCKS || process.env.APP_ENV === 'localElide' + ), + }; /* * ENV.APP.LOG_RESOLVER = true; * ENV.APP.LOG_ACTIVE_GENERATION = true; diff --git a/packages/data/addon/adapters/dimensions/elide.ts b/packages/data/addon/adapters/dimensions/elide.ts index bbf18e71b..ad184c9e6 100644 --- a/packages/data/addon/adapters/dimensions/elide.ts +++ b/packages/data/addon/adapters/dimensions/elide.ts @@ -117,7 +117,7 @@ export default class ElideDimensionAdapter extends EmberObject implements NaviDi operator: pred.operator, values: pred.values.map(String), })), - sorts: [], + sorts: [{ type: 'dimension', field: id, parameters, direction: 'asc' }], dataSource: source, limit: null, requestVersion: '2.0', diff --git a/packages/data/addon/adapters/facts/elide.ts b/packages/data/addon/adapters/facts/elide.ts index 66d5cfe00..9136b1ca6 100644 --- a/packages/data/addon/adapters/facts/elide.ts +++ b/packages/data/addon/adapters/facts/elide.ts @@ -69,6 +69,11 @@ export function getElideFilterField(fieldName: string, parameters: Parameters = return `${field}${paramsStr}`; } +type PaginationOptions = { + first: number; + after: number; +}; + export default class ElideFactsAdapter extends EmberObject implements NaviFactAdapter { /** * @property {Object} apollo - apollo client query manager using the overridden elide service @@ -91,7 +96,7 @@ export default class ElideFactsAdapter extends EmberObject implements NaviFactAd if (Object.keys(nonDefaultParams).length !== 0) { const canonicalName = canonicalizeMetric({ metric: base.field, parameters: base.parameters }); throw new FactAdapterError( - `Parameters are not supported in elide unles ${canonicalName} is added as a column.` + `Parameters are not supported in elide unless ${canonicalName} is added as a column.` ); } } @@ -170,7 +175,7 @@ export default class ElideFactsAdapter extends EmberObject implements NaviFactAd * @param request * @returns graphql query string for a v2 request */ - private dataQueryFromRequest(request: RequestV2): string { + private dataQueryFromRequest(request: RequestV2, pagination?: PaginationOptions | null): string { const args = []; const { table, columns, sorts, limit, filters } = request; const columnCanonicalToAlias = columns.reduce((canonicalToAlias: Record, column, idx) => { @@ -216,10 +221,22 @@ export default class ElideFactsAdapter extends EmberObject implements NaviFactAd const limitStr = limit ? `first: "${limit}"` : null; limitStr && args.push(limitStr); + if (pagination) { + pagination.first && args.push(`first: "${pagination.first}"`); + pagination.after && args.push(`after: "${pagination.after}"`); + } + const argsString = args.length ? `(${args.join(',')})` : ''; + let pageInfoString: string; + if (pagination === null) { + pageInfoString = ''; + } else { + pageInfoString = ' pageInfo { startCursor endCursor totalRecords }'; + } + return JSON.stringify({ - query: `{ ${table}${argsString} { edges { node { ${columnsStr} } } } }`, + query: `{ ${table}${argsString} { edges { node { ${columnsStr} } }${pageInfoString} } }`, }); } @@ -230,7 +247,20 @@ export default class ElideFactsAdapter extends EmberObject implements NaviFactAd */ createAsyncQuery(request: RequestV2, options: RequestOptions = {}): Promise { const mutation: DocumentNode = GQLQueries['asyncFactsMutation']; - const query = this.dataQueryFromRequest(request); + let pagination: PaginationOptions | undefined; + if (options.perPage) { + const page = options.page ?? 1; + if (request.limit && !(page === 1 && request.limit === options.perPage)) { + throw new FactAdapterError( + `The request specified a limit of ${request.limit} which conflicts with page=${page} and perPage=${options.perPage}` + ); + } + pagination = { + after: (page - 1) * options.perPage, + first: options.perPage, + }; + } + const query = this.dataQueryFromRequest(request, pagination); const asyncAfterSeconds = DEFAULT_ASYNC_AFTER_SECONDS; const id: string = options.requestId || v1(); const dataSourceName = request.dataSource || options.dataSourceName; @@ -265,7 +295,7 @@ export default class ElideFactsAdapter extends EmberObject implements NaviFactAd * @param _options */ urlForFindQuery(request: RequestV2, _options: RequestOptions): string { - return this.dataQueryFromRequest(request); + return this.dataQueryFromRequest(request, null); } /** diff --git a/packages/data/addon/adapters/facts/interface.ts b/packages/data/addon/adapters/facts/interface.ts index d43a73c76..c174f8376 100644 --- a/packages/data/addon/adapters/facts/interface.ts +++ b/packages/data/addon/adapters/facts/interface.ts @@ -89,6 +89,12 @@ export enum TableExportResultType { JSON = 'JSON', } +export type PageInfo = { + startCursor: `${number}`; + endCursor: `${number}`; + totalRecords: number; +}; + export type AsyncQueryResponse = { asyncQuery: { edges: [ diff --git a/packages/data/addon/mirage/routes/bard-lite.ts b/packages/data/addon/mirage/routes/bard-lite.ts index 342cae960..d4a7d354d 100644 --- a/packages/data/addon/mirage/routes/bard-lite.ts +++ b/packages/data/addon/mirage/routes/bard-lite.ts @@ -275,13 +275,14 @@ export default function ( this.get('/dimensions/:dimension/values', function (_db, request) { faker.seed(request.url.length); - let dimension = request.params.dimension, - rows = _getDimensionValues({ name: dimension, show: [] }); - + const dimension = request.params.dimension; + const { filters, page, perPage } = request.queryParams; + let rows = _getDimensionValues({ name: dimension, show: [] }); + let meta; // Handle value filters - if ('filters' in request.queryParams) { - const { values } = parseFilters(request.queryParams.filters)[0], - fieldMatch = request.queryParams.filters.match(/\|(id|key)/); + if (filters) { + const { values } = parseFilters(request.queryParams.filters)[0]; + const fieldMatch = request.queryParams.filters.match(/\|(id|key)/); rows = fieldMatch && fieldMatch.length > 0 @@ -291,8 +292,26 @@ export default function ( }) : rows.filter((row) => values.some((value) => row.description?.toLowerCase().includes(value.toLowerCase()))); } + if (page && perPage) { + const pageNum = Number(page); + const perPageNum = Number(perPage); + const skipped = (pageNum - 1) * perPageNum; + const totalResults = rows.length; + rows = rows.slice(skipped); + rows = rows.slice(0, perPageNum); + meta = { + pagination: { + currentPage: pageNum, + rowsPerPage: perPageNum, + numberOfResults: totalResults, + }, + }; + } - return { rows }; + return { + rows, + ...(meta ? { meta } : {}), + }; }); this.get('/dimensions/:dimension/search', function (_db, request) { diff --git a/packages/data/addon/mirage/routes/graphql.js b/packages/data/addon/mirage/routes/graphql.js index ec6869ac2..9211e369c 100644 --- a/packages/data/addon/mirage/routes/graphql.js +++ b/packages/data/addon/mirage/routes/graphql.js @@ -90,7 +90,10 @@ const DATE_FILTER_OPS = { */ function _getSeedForRequest(table, args, fields) { const tableLength = table.length; - const argsLength = Object.keys(args).join(' ').length; + const skippedArgs = ['first', 'after', 'sort']; + const argsLength = Object.keys(args) + .filter((key) => !skippedArgs.includes(key)) + .join(' ').length; const fieldsLength = fields.join(' ').length; return tableLength + argsLength + fieldsLength; } @@ -330,6 +333,7 @@ function _parseArgs(args, table, aliases) { return { field, direction }; }), first: (limit) => limit, + after: (after) => after, }; const parsed = {}; @@ -355,7 +359,7 @@ function _getResponseBody(db, asyncQueryRecord) { if (responseTime - createdOn >= ASYNC_RESPONSE_DELAY) { const { table, args, fields, aliases } = _parseGQLQuery(JSON.parse(query).query || ''); const fieldToAlias = invert(aliases); - const { filter = [], sort = [], first } = _parseArgs(args, table, aliases); + const { filter = [], sort = [], first, after } = _parseArgs(args, table, aliases); const seed = _getSeedForRequest(table, args, fields); faker.seed(seed); @@ -413,11 +417,6 @@ function _getResponseBody(db, asyncQueryRecord) { }, currRow) ); - // handle limit in request - if (first && first < rows.length) { - rows = rows.slice(0, first); - } - // sort rows if (sort.length) { rows = orderBy( @@ -427,10 +426,25 @@ function _getResponseBody(db, asyncQueryRecord) { ); } + // handle limit in request + const totalRecords = rows.length; + if (after && after < rows.length) { + rows = rows.slice(after); + } + if (first && first < rows.length) { + rows = rows.slice(0, first); + } + + const startNumber = after ?? 0; return JSON.stringify({ data: { [table]: { edges: rows.map((node) => ({ node })), + pageInfo: { + startCursor: `${startNumber}`, + endCursor: `${startNumber + rows.length}`, + totalRecords, + }, }, }, }); diff --git a/packages/data/addon/models/navi-dimension-response.ts b/packages/data/addon/models/navi-dimension-response.ts new file mode 100644 index 000000000..57e5faff0 --- /dev/null +++ b/packages/data/addon/models/navi-dimension-response.ts @@ -0,0 +1,12 @@ +/** + * Copyright 2021, Yahoo Holdings Inc. + * Licensed under the terms of the MIT license. See accompanying LICENSE.md file for terms. + */ +import EmberObject from '@ember/object'; +import type NaviDimensionModel from 'navi-data/models/navi-dimension'; +import type { ResponseV1 } from 'navi-data/serializers/facts/interface'; + +export default class NaviDimensionResponse extends EmberObject { + readonly values: NaviDimensionModel[] = []; + readonly meta?: ResponseV1['meta'] = {}; +} diff --git a/packages/data/addon/models/navi-facts.ts b/packages/data/addon/models/navi-facts.ts index d83e63141..e90c98412 100644 --- a/packages/data/addon/models/navi-facts.ts +++ b/packages/data/addon/models/navi-facts.ts @@ -6,10 +6,10 @@ */ import EmberObject from '@ember/object'; +import { taskFor } from 'ember-concurrency-ts'; import type NaviFactsService from 'navi-data/services/navi-facts'; import type { RequestV2 } from 'navi-data/adapters/facts/interface'; import type NaviFactResponse from 'navi-data/models/navi-fact-response'; -import { taskFor } from 'ember-concurrency-ts'; export default class NaviFacts extends EmberObject { /** @@ -28,16 +28,16 @@ export default class NaviFacts extends EmberObject { declare _factService: NaviFactsService; /** - * @returns Promise with the response model object for next page or null when trying to go past last page + * @returns Promise with the response model object for previous page or null when trying to access pages less than the first page */ - next(): Promise { - return taskFor(this._factService.fetchNext).perform(this.response, this.request); + previous(): Promise { + return taskFor(this._factService.fetchPrevious).perform(this.response, this.request); } /** - * @returns Promise with the response model object for previous page or null when trying to access pages less than the first page + * @returns Promise with the response model object for next page or null when trying to go past last page */ - previous(): Promise { + next(): Promise { return taskFor(this._factService.fetchNext).perform(this.response, this.request); } } diff --git a/packages/data/addon/serializers/dimensions/bard.ts b/packages/data/addon/serializers/dimensions/bard.ts index 098ca37b6..b19a34eb6 100644 --- a/packages/data/addon/serializers/dimensions/bard.ts +++ b/packages/data/addon/serializers/dimensions/bard.ts @@ -1,5 +1,5 @@ /** - * Copyright 2020, Yahoo Holdings Inc. + * Copyright 2021, Yahoo Holdings Inc. * Licensed under the terms of the MIT license. See accompanying LICENSE.md file for terms. */ @@ -8,17 +8,19 @@ import NaviDimensionSerializer from './interface'; import NaviDimensionModel from '../../models/navi-dimension'; import { FiliDimensionResponse, DefaultField } from 'navi-data/adapters/dimensions/bard'; import { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; export default class BardDimensionSerializer extends EmberObject implements NaviDimensionSerializer { - normalize(dimensionColumn: DimensionColumn, rawPayload: FiliDimensionResponse): NaviDimensionModel[] { + normalize(dimensionColumn: DimensionColumn, rawPayload: FiliDimensionResponse): NaviDimensionResponse { if (rawPayload?.rows.length) { const field = dimensionColumn.parameters?.field || DefaultField; - return rawPayload.rows.map((row) => { + const values = rawPayload.rows.map((row) => { //TODO remove when https://github.com/yahoo/fili/issues/1088 lands const value = 'desc' === field ? row.description : row[field]; return NaviDimensionModel.create({ value, dimensionColumn }); }); + return NaviDimensionResponse.create({ values, meta: rawPayload.meta }); } - return []; + return NaviDimensionResponse.create(); } } diff --git a/packages/data/addon/serializers/dimensions/elide.ts b/packages/data/addon/serializers/dimensions/elide.ts index 4fc2f05ad..2286b2322 100644 --- a/packages/data/addon/serializers/dimensions/elide.ts +++ b/packages/data/addon/serializers/dimensions/elide.ts @@ -2,32 +2,47 @@ * Copyright 2021, Yahoo Holdings Inc. * Licensed under the terms of the MIT license. See accompanying LICENSE.md file for terms. */ - -import NaviDimensionSerializer from './interface'; -import NaviDimensionModel from '../../models/navi-dimension'; -import { AsyncQueryResponse } from 'navi-data/adapters/facts/interface'; import EmberObject from '@ember/object'; -import { DimensionColumn } from 'navi-data/models/metadata/dimension'; -import ElideDimensionMetadataModel from 'navi-data/models/metadata/elide/dimension'; +import { assert } from '@ember/debug'; +import NaviDimensionModel from '../../models/navi-dimension'; +import NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; +import { getPaginationFromPageInfo } from '../facts/elide'; +import type { AsyncQueryResponse } from 'navi-data/adapters/facts/interface'; +import type NaviDimensionSerializer from './interface'; +import type { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import type ElideDimensionMetadataModel from 'navi-data/models/metadata/elide/dimension'; +import type { ServiceOptions } from 'navi-data/services/navi-dimension'; export type ResponseEdge = { node: Record; }; export default class ElideDimensionSerializer extends EmberObject implements NaviDimensionSerializer { - normalize(dimension: DimensionColumn, rawPayload?: AsyncQueryResponse): NaviDimensionModel[] { + normalize( + dimension: DimensionColumn, + rawPayload?: AsyncQueryResponse, + options: ServiceOptions = {} + ): NaviDimensionResponse { const responseStr = rawPayload?.asyncQuery.edges[0].node.result?.responseBody; const { tableId } = (dimension.columnMetadata as ElideDimensionMetadataModel).lookupColumn; + assert('The tableId is defined', tableId); if (responseStr) { const response = JSON.parse(responseStr); - return response.data[tableId as string].edges.map((edge: ResponseEdge) => + const { edges, pageInfo } = response.data[tableId]; + const values = edges.map((edge: ResponseEdge) => NaviDimensionModel.create({ value: edge.node.col0, dimensionColumn: dimension, }) ); + return NaviDimensionResponse.create({ + values, + meta: { + pagination: getPaginationFromPageInfo(pageInfo, options), + }, + }); } - return []; + return NaviDimensionResponse.create(); } } diff --git a/packages/data/addon/serializers/dimensions/interface.ts b/packages/data/addon/serializers/dimensions/interface.ts index bfe50620a..9d6b827de 100644 --- a/packages/data/addon/serializers/dimensions/interface.ts +++ b/packages/data/addon/serializers/dimensions/interface.ts @@ -1,11 +1,12 @@ /** - * Copyright 2020, Yahoo Holdings Inc. + * Copyright 2021, Yahoo Holdings Inc. * Licensed under the terms of the MIT license. See accompanying LICENSE.md file for terms. */ -import NaviDimensionModel from '../../models/navi-dimension'; import EmberObject from '@ember/object'; -import { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import type { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import type { ServiceOptions } from 'navi-data/services/navi-dimension'; +import type NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; export default interface NaviDimensionSerializer extends EmberObject { - normalize(dimension: DimensionColumn, rawPayload: unknown): NaviDimensionModel[]; + normalize(dimension: DimensionColumn, rawPayload: unknown, options: ServiceOptions): NaviDimensionResponse; } diff --git a/packages/data/addon/serializers/facts/elide.ts b/packages/data/addon/serializers/facts/elide.ts index 63f644be4..2f150de9c 100644 --- a/packages/data/addon/serializers/facts/elide.ts +++ b/packages/data/addon/serializers/facts/elide.ts @@ -6,8 +6,14 @@ */ import EmberObject from '@ember/object'; -import NaviFactSerializer from './interface'; -import { AsyncQueryResponse, FactAdapterError, RequestV2 } from 'navi-data/adapters/facts/interface'; +import NaviFactSerializer, { ResponseV1 } from './interface'; +import { + AsyncQueryResponse, + FactAdapterError, + PageInfo, + RequestOptions, + RequestV2, +} from 'navi-data/adapters/facts/interface'; import { canonicalizeMetric } from 'navi-data/utils/metric'; import NaviFactResponse from 'navi-data/models/navi-fact-response'; import NaviFactError, { NaviErrorDetails } from 'navi-data/errors/navi-adapter-error'; @@ -23,12 +29,37 @@ function isElideResponse(response?: AsyncQueryResponse | ExecutionResult | Error function isApolloError(response?: AsyncQueryResponse | ExecutionResult | Error): response is ExecutionResult { return (response as ExecutionResult)?.errors !== undefined; } + +interface PaginationOptions { + perPage?: number; +} +export function getPaginationFromPageInfo( + pageInfo?: PageInfo, + options?: PaginationOptions +): ResponseV1['meta']['pagination'] { + if (!pageInfo) { + return undefined; + } + const startCursor = Number(pageInfo.startCursor); + const endCursor = Number(pageInfo.endCursor); + // Use the perPage property if available (e.g. Getting the last page but there's only 2 results left) + const rowsPerPage = options?.perPage ?? endCursor - startCursor; + + // Integer division of start position and page size (indexing starting with 1) + const currentPage = rowsPerPage !== 0 ? Math.floor(startCursor / rowsPerPage) + 1 : 1; + return { + rowsPerPage, + currentPage, + numberOfResults: pageInfo.totalRecords, + }; +} + export default class ElideFactsSerializer extends EmberObject implements NaviFactSerializer { /** * @param payload - raw payload string * @param request - request object */ - private processResponse(payload: string, request: RequestV2): NaviFactResponse { + private processResponse(payload: string, request: RequestV2, options: RequestOptions): NaviFactResponse { const response = JSON.parse(payload) as ExecutionResult; const { table } = request; const elideFields = request.columns.map((_c, idx) => `col${idx}`); @@ -39,6 +70,7 @@ export default class ElideFactsSerializer extends EmberObject implements NaviFac const { data } = response; assert('`data` should be present in successful in a response', data); + const pageInfo = data[table].pageInfo; const rawRows = data[table].edges; const totalRows = rawRows.length; const totalFields = normalizedFields.length; @@ -54,15 +86,28 @@ export default class ElideFactsSerializer extends EmberObject implements NaviFac } } - return NaviFactResponse.create({ rows, meta: {} }); + return NaviFactResponse.create({ + rows, + meta: { + pagination: getPaginationFromPageInfo(pageInfo, options), + }, + }); } - normalize(payload: AsyncQueryResponse, request: RequestV2): NaviFactResponse | undefined { + normalize( + payload: AsyncQueryResponse, + request: RequestV2, + options: RequestOptions = {} + ): NaviFactResponse | undefined { const responseStr = payload?.asyncQuery.edges[0].node.result?.responseBody; - return responseStr ? this.processResponse(responseStr, request) : undefined; + return responseStr ? this.processResponse(responseStr, request, options) : undefined; } - extractError(payload: ExecutionResult | AsyncQueryResponse | Error, _request: RequestV2): NaviFactError { + extractError( + payload: ExecutionResult | AsyncQueryResponse | Error, + _request: RequestV2, + _options: RequestOptions + ): NaviFactError { let errors: NaviErrorDetails[] = []; if (isElideResponse(payload)) { const responseStr = payload.asyncQuery.edges[0].node.result?.responseBody; diff --git a/packages/data/addon/serializers/facts/interface.ts b/packages/data/addon/serializers/facts/interface.ts index 802b8947e..1b99eb888 100644 --- a/packages/data/addon/serializers/facts/interface.ts +++ b/packages/data/addon/serializers/facts/interface.ts @@ -1,11 +1,10 @@ /** - * Copyright 2020, Yahoo Holdings Inc. + * Copyright 2021, Yahoo Holdings Inc. * Licensed under the terms of the MIT license. See accompanying LICENSE.md file for terms. */ - -import { RequestV1, RequestV2 } from 'navi-data/adapters/facts/interface'; -import NaviFactResponse from 'navi-data/models/navi-fact-response'; -import NaviAdapterError from 'navi-data/errors/navi-adapter-error'; +import type { RequestOptions, RequestV2 } from 'navi-data/adapters/facts/interface'; +import type NaviFactResponse from 'navi-data/models/navi-fact-response'; +import type NaviAdapterError from 'navi-data/errors/navi-adapter-error'; export interface ResponseV1 { readonly rows: Array>; @@ -13,7 +12,6 @@ export interface ResponseV1 { pagination?: { currentPage: number; rowsPerPage: number; - perPage: number; numberOfResults: number; }; }; @@ -25,12 +23,12 @@ export default interface NaviFactSerializer { * @param payload - payload to normalize * @param request - request for response payload */ - normalize(payload: unknown, request: RequestV1 | RequestV2): NaviFactResponse | undefined; + normalize(payload: unknown, request: RequestV2, options?: RequestOptions): NaviFactResponse | undefined; /** * Extract errors from server * @param payload - payload to normalize * @param request - request for response payload */ - extractError(payload: unknown, request: RequestV1 | RequestV2): NaviAdapterError; + extractError(payload: unknown, request: RequestV2, options?: RequestOptions): NaviAdapterError; } diff --git a/packages/data/addon/serializers/metadata/bard.ts b/packages/data/addon/serializers/metadata/bard.ts index 861e8ed12..1de5eb43f 100644 --- a/packages/data/addon/serializers/metadata/bard.ts +++ b/packages/data/addon/serializers/metadata/bard.ts @@ -25,8 +25,8 @@ import { Grain } from 'navi-data/utils/date'; import { getOwner } from '@ember/application'; import { sortBy } from 'lodash-es'; -const LOAD_CARDINALITY = config.navi.searchThresholds.contains; -const MAX_LOAD_CARDINALITY = config.navi.searchThresholds.in; +const SMALL_CARDINALITY = config.navi.cardinalities.small; +const MEDIUM_CARDINALITY = config.navi.cardinalities.medium; export type RawEverythingPayload = { tables: RawTablePayload[]; @@ -531,11 +531,13 @@ export default class BardMetadataSerializer extends NaviMetadataSerializer { fields, } = dimension; - let dimCardinality: Cardinality = CARDINALITY_SIZES[0]; - if (cardinality > MAX_LOAD_CARDINALITY) { + let dimCardinality: Cardinality; + if (cardinality > MEDIUM_CARDINALITY) { dimCardinality = CARDINALITY_SIZES[2]; - } else if (cardinality > LOAD_CARDINALITY) { + } else if (cardinality > SMALL_CARDINALITY) { dimCardinality = CARDINALITY_SIZES[1]; + } else { + dimCardinality = CARDINALITY_SIZES[0]; } return { id: name, diff --git a/packages/data/addon/services/navi-dimension.ts b/packages/data/addon/services/navi-dimension.ts index d7daea638..29db5901c 100644 --- a/packages/data/addon/services/navi-dimension.ts +++ b/packages/data/addon/services/navi-dimension.ts @@ -11,8 +11,9 @@ import type { TaskGenerator } from 'ember-concurrency'; import type NaviDimensionSerializer from 'navi-data/serializers/dimensions/interface'; import type NaviDimensionAdapter from 'navi-data/adapters/dimensions/interface'; import type { DimensionFilter } from 'navi-data/adapters/dimensions/interface'; -import type NaviDimensionModel from 'navi-data/models/navi-dimension'; import type { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; +import NaviDimensionModel from 'navi-data/models/navi-dimension'; export type ServiceOptions = { timeout?: number; @@ -39,15 +40,39 @@ export default class NaviDimensionService extends Service { } /** - * Get all values for a dimension column + * Get all values for a dimension column, paginating through results as needed. * @param dimension - requested dimension * @param options - method options */ - @task *all(dimension: DimensionColumn, options?: ServiceOptions): TaskGenerator { + @task *all(dimension: DimensionColumn, options: ServiceOptions = {}): TaskGenerator { const { type: dataSourceType } = getDataSource(dimension.columnMetadata.source); const adapter = this.adapterFor(dataSourceType); - const payload: unknown = yield taskFor(adapter.all).perform(dimension, options); - return this.serializerFor(dataSourceType).normalize(dimension, payload); + const serializer = this.serializerFor(dataSourceType); + let moreResults = true; + let paginationOptions: Pick = {}; + let values: NaviDimensionModel[] = []; + while (moreResults) { + const mergedOptions = { ...options, ...paginationOptions }; + const payload: unknown = yield taskFor(adapter.all).perform(dimension, mergedOptions); + const normalized = serializer.normalize(dimension, payload, mergedOptions); + + values.push(...normalized.values); + + if (normalized.meta?.pagination) { + // Pagination data found, check if more results are available + const { currentPage, numberOfResults, rowsPerPage } = normalized.meta.pagination; + const currentCount = currentPage * rowsPerPage; + moreResults = currentCount < numberOfResults; + paginationOptions = { + page: currentPage + 1, + perPage: rowsPerPage, + }; + } else { + // No pagination data, assume all values were fetched + moreResults = false; + } + } + return NaviDimensionResponse.create({ values }); } /** @@ -59,12 +84,12 @@ export default class NaviDimensionService extends Service { @task *find( dimension: DimensionColumn, predicate: DimensionFilter[], - options?: ServiceOptions - ): TaskGenerator { + options: ServiceOptions = {} + ): TaskGenerator { const { type: dataSourceType } = getDataSource(dimension.columnMetadata.source); const adapter = this.adapterFor(dataSourceType); const payload: unknown = yield taskFor(adapter.find).perform(dimension, predicate, options); - return this.serializerFor(dataSourceType).normalize(dimension, payload); + return this.serializerFor(dataSourceType).normalize(dimension, payload, options); } /** @@ -76,12 +101,12 @@ export default class NaviDimensionService extends Service { @task *search( dimension: DimensionColumn, query: string, - options?: ServiceOptions - ): TaskGenerator { + options: ServiceOptions = {} + ): TaskGenerator { const { type: dataSourceType } = getDataSource(dimension.columnMetadata.source); const adapter = this.adapterFor(dataSourceType); const payload: unknown = yield taskFor(adapter.search).perform(dimension, query, options); - return this.serializerFor(dataSourceType).normalize(dimension, payload); + return this.serializerFor(dataSourceType).normalize(dimension, payload, options); } } diff --git a/packages/data/addon/services/navi-facts.ts b/packages/data/addon/services/navi-facts.ts index a57ac36c2..bc63bc097 100644 --- a/packages/data/addon/services/navi-facts.ts +++ b/packages/data/addon/services/navi-facts.ts @@ -16,7 +16,7 @@ import type { TaskGenerator } from 'ember-concurrency'; import type NaviFactAdapter from 'navi-data/adapters/facts/interface'; import type { RequestOptions, RequestV2 } from 'navi-data/adapters/facts/interface'; import type NaviFactSerializer from 'navi-data/serializers/facts/interface'; -import type { ResponseV1 } from 'navi-data/serializers/facts/interface'; +import type NaviFactResponse from 'navi-data/models/navi-fact-response'; export default class NaviFactsService extends Service { /** @@ -90,10 +90,10 @@ export default class NaviFactsService extends Service { try { const payload: unknown = yield taskFor(adapter.fetchDataForRequest).perform(request, options); - const response = serializer.normalize(payload, request); + const response = serializer.normalize(payload, request, options); return NaviFactsModel.create({ request, response, _factService: this }); } catch (e) { - const errorModel: Error = serializer.extractError(e, request); + const errorModel: Error = serializer.extractError(e, request, options); throw errorModel; } } @@ -103,15 +103,15 @@ export default class NaviFactsService extends Service { * @param request * @return returns the promise with the next set of results or null */ - @task *fetchNext(response: ResponseV1, request: RequestV2): TaskGenerator { + @task *fetchNext(response: NaviFactResponse, request: RequestV2): TaskGenerator { if (response.meta.pagination) { - const { perPage, numberOfResults, currentPage } = response.meta.pagination; - const totalPages = numberOfResults / perPage; + const { rowsPerPage, numberOfResults, currentPage } = response.meta.pagination; + const totalPages = numberOfResults / rowsPerPage; if (currentPage < totalPages) { return yield taskFor(this.fetch).perform(request, { page: currentPage + 1, - perPage: perPage, + perPage: rowsPerPage, }); } } @@ -123,7 +123,7 @@ export default class NaviFactsService extends Service { * @param request * @return returns the promise with the previous set of results or null */ - @task *fetchPrevious(response: ResponseV1, request: RequestV2): TaskGenerator { + @task *fetchPrevious(response: NaviFactResponse, request: RequestV2): TaskGenerator { if (response.meta.pagination) { const { rowsPerPage, currentPage } = response.meta.pagination; if (currentPage > 1) { diff --git a/packages/data/app/models/navi-dimension-response.js b/packages/data/app/models/navi-dimension-response.js new file mode 100644 index 000000000..fb99db6c4 --- /dev/null +++ b/packages/data/app/models/navi-dimension-response.js @@ -0,0 +1 @@ +export { default } from 'navi-data/models/navi-dimension-response'; diff --git a/packages/data/config/environment.js b/packages/data/config/environment.js index d72a42b5f..88a832f8e 100644 --- a/packages/data/config/environment.js +++ b/packages/data/config/environment.js @@ -4,9 +4,9 @@ module.exports = function (/* environment, appConfig */) { return { navi: { dataEpoch: '2013-01-01', - searchThresholds: { - contains: 600, - in: 50000, + cardinalities: { + small: 600, + medium: 50000, }, }, }; diff --git a/packages/data/config/navi-config.d.ts b/packages/data/config/navi-config.d.ts index 6ebc1f7db..c99fe9dd0 100644 --- a/packages/data/config/navi-config.d.ts +++ b/packages/data/config/navi-config.d.ts @@ -11,7 +11,10 @@ declare module 'navi-config' { dataEpoch: string; dataSources: NaviDataSource[]; defaultDataSource?: string; - searchThresholds: TODO; + cardinalities: { + small: number; + medium: number; + }; defaultTimeGrain?: Grain; } } diff --git a/packages/data/tests/dummy/config/environment.js b/packages/data/tests/dummy/config/environment.js index d1ec65c27..4641e7540 100644 --- a/packages/data/tests/dummy/config/environment.js +++ b/packages/data/tests/dummy/config/environment.js @@ -32,9 +32,9 @@ module.exports = function (environment) { { name: 'elideTwo', uri: 'https://data2.naviapp.com/graphql', type: 'elide' }, ], defaultDataSource: 'bardOne', - searchThresholds: { - contains: 600, - in: 50000, + cardinalities: { + small: 600, + medium: 50000, }, }, }; diff --git a/packages/data/tests/unit/adapters/dimensions/elide-test.ts b/packages/data/tests/unit/adapters/dimensions/elide-test.ts index be8e29930..b9ffdd487 100644 --- a/packages/data/tests/unit/adapters/dimensions/elide-test.ts +++ b/packages/data/tests/unit/adapters/dimensions/elide-test.ts @@ -84,7 +84,14 @@ module('Unit | Adapter | Dimensions | Elide', function (hooks) { values: ['v1', 'v2'], }, ], - sorts: [], + sorts: [ + { + type: 'dimension', + field: 'table0.dimension0', + parameters: { foo: 'bar' }, + direction: 'asc', + }, + ], table: 'table0', limit: null, dataSource: 'elideTwo', @@ -194,7 +201,14 @@ module('Unit | Adapter | Dimensions | Elide', function (hooks) { values: ['v1', 'v2'], }, ], - sorts: [], + sorts: [ + { + type: 'dimension', + field: 'table0.dimension0', + parameters: {}, + direction: 'asc', + }, + ], table: lookupDimColumn.split('.')[0], limit: null, dataSource: 'elideTwo', @@ -225,7 +239,14 @@ module('Unit | Adapter | Dimensions | Elide', function (hooks) { const expectedRequest: RequestV2 = { columns: [{ field: 'table0.dimension0', parameters: { foo: 'baz' }, type: 'dimension' }], filters: [], - sorts: [], + sorts: [ + { + type: 'dimension', + field: 'table0.dimension0', + parameters: { foo: 'baz' }, + direction: 'asc', + }, + ], table: 'table0', limit: null, dataSource: 'elideTwo', @@ -304,7 +325,14 @@ module('Unit | Adapter | Dimensions | Elide', function (hooks) { values: ['*something*'], }, ], - sorts: [], + sorts: [ + { + direction: 'asc', + field: 'table0.dimension0', + parameters: { bang: 'boom' }, + type: 'dimension', + }, + ], table: 'table0', limit: null, dataSource: 'elideOne', diff --git a/packages/data/tests/unit/adapters/facts/elide-test.ts b/packages/data/tests/unit/adapters/facts/elide-test.ts index 1fe5ae480..16a0be907 100644 --- a/packages/data/tests/unit/adapters/facts/elide-test.ts +++ b/packages/data/tests/unit/adapters/facts/elide-test.ts @@ -83,7 +83,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { const queryStr = adapter['dataQueryFromRequest'](TestRequest); assert.equal( queryStr, - `{"query":"{ table1(filter: \\"d3=in=('v1','v2');d4=in=('v3','v4');d5=isnull=true;time[grain:day]=ge=('2015-01-03');time[grain:day]=lt=('2015-01-04');col0=gt=('0')\\",sort: \\"col3\\",first: \\"10000\\") { edges { node { col0:m1 col1:m2 col2:r(p:\\"123\\") col3:d1 col4:d2 } } } }"}`, + `{"query":"{ table1(filter: \\"d3=in=('v1','v2');d4=in=('v3','v4');d5=isnull=true;time[grain:day]=ge=('2015-01-03');time[grain:day]=lt=('2015-01-04');col0=gt=('0')\\",sort: \\"col3\\",first: \\"10000\\") { edges { node { col0:m1 col1:m2 col2:r(p:\\"123\\") col3:d1 col4:d2 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'dataQueryFromRequestV2 returns the correct query string for the given request V2' ); @@ -100,7 +100,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { requestVersion: '2.0', dataSource: 'elideOne', }), - `{"query":"{ myTable { edges { node { col0:m1(p:\\"q\\") col1:d1 } } } }"}`, + `{"query":"{ myTable { edges { node { col0:m1(p:\\"q\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Arguments are properly excluded if they are not in the request' ); @@ -121,7 +121,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { requestVersion: '2.0', dataSource: 'elideOne', }), - `{"query":"{ myTable(sort: \\"-col1,col2\\") { edges { node { col0:m1 col1:m1(p:\\"q\\") col2:d1 } } } }"}`, + `{"query":"{ myTable(sort: \\"-col1,col2\\") { edges { node { col0:m1 col1:m1(p:\\"q\\") col2:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Request with sorts and parameters is queried correctly' ); @@ -143,7 +143,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col1=in=('v1','v2');col2!=('a');d2==('b')\\") { edges { node { col0:m1 col1:m1(p:\\"q\\") col2:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col1=in=('v1','v2');col2!=('a');d2==('b')\\") { edges { node { col0:m1 col1:m1(p:\\"q\\") col2:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Request with filters and parameters is queried correctly' ); @@ -160,7 +160,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { requestVersion: '2.0', dataSource: 'elideOne', }), - `{"query":"{ myTable(first: \\"5\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(first: \\"5\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Request with limit is queried correctly' ); @@ -179,7 +179,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col0=ge=('v1');col0=le=('v2')\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col0=ge=('v1');col0=le=('v2')\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Request with "between" filter operator splits the filter into two correctly' ); @@ -198,10 +198,11 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col0=lt=('v1'),col0=gt=('v2')\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col0=lt=('v1'),col0=gt=('v2')\\") { edges { node { col0:m1(p:\\"q\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Request with "not between" filter operator splits the filter into two correctly' ); + const lastMonth = moment.utc().subtract(1, 'month').format('YYYY-MM'); assert.equal( adapter['dataQueryFromRequest']({ table: 'myTable', @@ -223,11 +224,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col0=ge=('${moment() - .subtract(1, 'month') - .format('YYYY-MM')}');col0=le=('${moment() - .subtract(1, 'month') - .format('YYYY-MM')}')\\") { edges { node { col0:time(grain:\\"MONTH\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col0=ge=('${lastMonth}');col0=le=('${lastMonth}')\\") { edges { node { col0:time(grain:\\"MONTH\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Macros and durations in time-dimension filters are converted to date strings properly ([P1X, current] -> equals 1 X duration)' ); @@ -252,7 +249,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col0=isnull=true\\") { edges { node { col0:time(grain:\\"DAY\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col0=isnull=true\\") { edges { node { col0:time(grain:\\"DAY\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Filter without 2 filter values is unaffected' ); @@ -277,9 +274,59 @@ module('Unit | Adapter | facts/elide', function (hooks) { dataSource: 'elideOne', limit: null, }), - `{"query":"{ myTable(filter: \\"col0=ge=('2020-05-05');col0=le=('2020-05-08')\\") { edges { node { col0:time(grain:\\"DAY\\") col1:d1 } } } }"}`, + `{"query":"{ myTable(filter: \\"col0=ge=('2020-05-05');col0=le=('2020-05-08')\\") { edges { node { col0:time(grain:\\"DAY\\") col1:d1 } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'Filter with 2 non-macro date values is unaffected' ); + + assert.strictEqual( + adapter['dataQueryFromRequest']({ ...TestRequest, limit: null }, { first: 10, after: 1 }), + `{"query":"{ table1(filter: \\"d3=in=('v1','v2');d4=in=('v3','v4');d5=isnull=true;time[grain:day]=ge=('2015-01-03');time[grain:day]=lt=('2015-01-04');col0=gt=('0')\\",sort: \\"col3\\",first: \\"10\\",after: \\"1\\") { edges { node { col0:m1 col1:m2 col2:r(p:\\"123\\") col3:d1 col4:d2 } } pageInfo { startCursor endCursor totalRecords } } }"}`, + 'The first and after pagination options are applied to the query' + ); + }); + + test('createAsyncQuery - pagination', async function (assert) { + assert.expect(7); + const adapter: ElideFactsAdapter = this.owner.lookup('adapter:facts/elide'); + + let Pagination: Parameters[1] = undefined; + let Message = ''; + adapter.apollo.mutate = () => undefined; + adapter['dataQueryFromRequest'] = (_request, pagination) => { + assert.deepEqual(pagination, Pagination, Message); + return ''; + }; + + assert.throws( + () => adapter.createAsyncQuery(TestRequest, { perPage: 2 }), + /The request specified a limit of 10000 which conflicts with page=1 and perPage=2/, + 'The request cannot specify a different limit and perPage' + ); + + assert.throws( + () => adapter.createAsyncQuery(TestRequest, { perPage: 1000, page: 2 }), + /The request specified a limit of 10000 which conflicts with page=2 and perPage=1000/, + 'The request cannot specify a different limit and perPage' + ); + + Pagination = { first: 10000, after: 0 }; + Message = 'A limit and perPage that are equal is allowed'; + await adapter.createAsyncQuery(TestRequest, { perPage: 10000 }); + + Pagination = { first: 10000, after: 0 }; + Message = 'A limit and perPage that are equal is allowed if page=1'; + await adapter.createAsyncQuery(TestRequest, { perPage: 10000, page: 1 }); + + const limitless = { ...TestRequest, limit: null }; + Pagination = { first: 3, after: 3 }; + Message = 'Specifying perPage and page is translated correctly'; + await adapter.createAsyncQuery(limitless, { perPage: 3, page: 2 }); + + Pagination = { first: 4, after: 12 }; + await adapter.createAsyncQuery(limitless, { perPage: 4, page: 4 }); + + Pagination = { first: 4, after: 4 }; + await adapter.createAsyncQuery(limitless, { perPage: 4, page: 2 }); }); test('createAsyncQuery - success', async function (assert) { @@ -307,7 +354,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { assert.equal( requestObj.variables.query.replace(/[ \t\r\n]+/g, ' '), JSON.stringify({ - query: `{ ${expectedTable}${expectedArgs} { edges { node { ${expectedColumns} } } } }`, + query: `{ ${expectedTable}${expectedArgs} { edges { node { ${expectedColumns} } } pageInfo { startCursor endCursor totalRecords } } }`, }).replace(/[ \t\r\n]+/g, ' '), 'createAsyncQuery sends the correct query variable string' ); @@ -744,7 +791,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { //test all of the escaped functionalities and verify them in the below assert assert.equal( queryStr, - `{"query":"{ table1(filter: \\"d6[field:id]=in=('with, comma','no comma');d7[field:id]=in=('with \\"quote\\"','but why');d8[field:id]=in=('okay','with \\\\\\\\'single quote\\\\\\\\'')\\",sort: \\"d1\\",first: \\"10000\\") { edges { node { } } } }"}`, + `{"query":"{ table1(filter: \\"d6[field:id]=in=('with, comma','no comma');d7[field:id]=in=('with \\"quote\\"','but why');d8[field:id]=in=('okay','with \\\\\\\\'single quote\\\\\\\\'')\\",sort: \\"d1\\",first: \\"10000\\") { edges { node { } } pageInfo { startCursor endCursor totalRecords } } }"}`, 'dataQueryFromRequestV2 returns the correct query string with escaped quotes and commas for the given request V2' ); }); @@ -891,7 +938,7 @@ module('Unit | Adapter | facts/elide', function (hooks) { assert.equal( decodeURIComponent(adapter.urlForFindQuery(TestRequest, {})), `{"query":"{ table1(filter: \\"d3=in=('v1','v2');d4=in=('v3','v4');d5=isnull=true;time[grain:day]=ge=('2015-01-03');time[grain:day]=lt=('2015-01-04');col0=gt=('0')\\",sort: \\"col3\\",first: \\"10000\\") { edges { node { col0:m1 col1:m2 col2:r(p:\\"123\\") col3:d1 col4:d2 } } } }"}`, - 'urlForFindQuery correctly built the query for the provided request' + 'urlForFindQuery correctly built the query for the provided request with no pagination info' ); }); }); diff --git a/packages/data/tests/unit/models/navi-dimension-response-test.ts b/packages/data/tests/unit/models/navi-dimension-response-test.ts new file mode 100644 index 000000000..b206169ed --- /dev/null +++ b/packages/data/tests/unit/models/navi-dimension-response-test.ts @@ -0,0 +1,13 @@ +import { module, test } from 'qunit'; +import { setupTest } from 'ember-qunit'; +import NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; + +module('Unit | Model | navi dimension response', function (hooks) { + setupTest(hooks); + + test('it exists', function (assert) { + const response = NaviDimensionResponse.create(); + assert.deepEqual(response.values, [], 'Stores dimension values'); + assert.deepEqual(response.meta, {}, 'Stores meta information'); + }); +}); diff --git a/packages/data/tests/unit/serializers/dimensions/bard-test.ts b/packages/data/tests/unit/serializers/dimensions/bard-test.ts index 46040b757..afeb6094c 100644 --- a/packages/data/tests/unit/serializers/dimensions/bard-test.ts +++ b/packages/data/tests/unit/serializers/dimensions/bard-test.ts @@ -39,12 +39,12 @@ module('Unit | Serializer | Dimensions | Bard', function (hooks) { const normalized = this.serializer.normalize(dimColumn, payload); assert.deepEqual( - normalized.map(({ value }) => value), + normalized.values.map(({ value }) => value), ContainerDimValues.map(({ id }) => id), '`normalize` hydrated NaviDimensionModel objects with the correct field value' ); assert.deepEqual( - normalized.map(({ displayValue }) => displayValue), + normalized.values.map(({ displayValue }) => displayValue), ContainerDimValues.map(({ id }) => id), '`normalize` hydrated NaviDimensionModel and produced the correct `displayValue`' ); @@ -65,13 +65,13 @@ module('Unit | Serializer | Dimensions | Bard', function (hooks) { const normalized = this.serializer.normalize(dimColumn, payload); assert.deepEqual( - normalized.map(({ value }) => value), + normalized.values.map(({ value }) => value), ContainerDimValues.map(({ description }) => description), '`normalize` uses the `description` property when `desc` field is requested' ); assert.deepEqual( - normalized.map(({ displayValue }) => displayValue), + normalized.values.map(({ displayValue }) => displayValue), ContainerDimValues.map(({ description }) => description), '`normalize` hydrated NaviDimensionModel and produced the correct `displayValue`' ); @@ -86,6 +86,10 @@ module('Unit | Serializer | Dimensions | Bard', function (hooks) { field: 'description', }, }; - assert.deepEqual(this.serializer.normalize(dimColumn, payload), [], '`normalize` can handle an empty payload'); + assert.deepEqual( + this.serializer.normalize(dimColumn, payload).values, + [], + '`normalize` can handle an empty payload' + ); }); }); diff --git a/packages/data/tests/unit/serializers/dimensions/elide-test.ts b/packages/data/tests/unit/serializers/dimensions/elide-test.ts index d53bd9994..bbf588fbc 100644 --- a/packages/data/tests/unit/serializers/dimensions/elide-test.ts +++ b/packages/data/tests/unit/serializers/dimensions/elide-test.ts @@ -41,8 +41,18 @@ module('Unit | Serializer | Dimensions | Elide', function (hooks) { contentLength: 129, httpStatus: 200, recordCount: 3, - responseBody: - '{"data":{"table0":{"edges":[{"node":{"col0":"foo"}},{"node":{"col0":"bar"}},{"node":{"col0":"baz"}}]}}}', + responseBody: JSON.stringify({ + data: { + table0: { + edges: [{ node: { col0: 'foo' } }, { node: { col0: 'bar' } }, { node: { col0: 'baz' } }], + pageInfo: { + startCursor: '0', + endCursor: '3', + totalRecords: 6, + }, + }, + }, + }), }, }, }, @@ -56,11 +66,30 @@ module('Unit | Serializer | Dimensions | Elide', function (hooks) { 'elideOne' ) as DimensionMetadataModel, }; - assert.deepEqual(serializer.normalize(dimensionColumn), [], 'Empty array is returned for an undefined payload'); + assert.deepEqual( + serializer.normalize(dimensionColumn).values, + [], + 'Empty array is returned for an undefined payload' + ); const expectedModels = ['foo', 'bar', 'baz'].map((value) => NaviDimensionModel.create({ value, dimensionColumn })); - const actualModels = serializer.normalize(dimensionColumn, payload); - assert.deepEqual(actualModels, expectedModels, 'normalize returns the `rows` prop of the raw payload'); + const dimensionResponse = serializer.normalize(dimensionColumn, payload); + assert.deepEqual( + dimensionResponse.values, + expectedModels, + 'normalize returns the dimension values of the raw payload' + ); + assert.deepEqual( + dimensionResponse.meta, + { + pagination: { + currentPage: 1, + numberOfResults: 6, + rowsPerPage: 3, + }, + }, + 'normalize creates a `meta` prop for pagination' + ); }); test('normalize - tableSource', function (this: TestContext, assert) { @@ -82,7 +111,18 @@ module('Unit | Serializer | Dimensions | Elide', function (hooks) { contentLength: 129, httpStatus: 200, recordCount: 3, - responseBody: `{"data":{"${lookupTable}":{"edges":[{"node":{"col0":"foo"}},{"node":{"col0":"bar"}},{"node":{"col0":"baz"}}]}}}`, + responseBody: JSON.stringify({ + data: { + [lookupTable]: { + edges: [{ node: { col0: 'foo' } }, { node: { col0: 'bar' } }, { node: { col0: 'baz' } }], + pageInfo: { + startCursor: '0', + endCursor: '3', + totalRecords: 6, + }, + }, + }, + }), }, }, }, @@ -97,11 +137,22 @@ module('Unit | Serializer | Dimensions | Elide', function (hooks) { ) as DimensionMetadataModel, }; const expectedModels = ['foo', 'bar', 'baz'].map((value) => NaviDimensionModel.create({ value, dimensionColumn })); - const actualModels = serializer.normalize(dimensionColumn, payload); + const dimensionResponse = serializer.normalize(dimensionColumn, payload); assert.deepEqual( - actualModels, + dimensionResponse.values, expectedModels, '`tableSource`, when available, is used to normalize dimension value responses' ); + assert.deepEqual( + dimensionResponse.meta, + { + pagination: { + currentPage: 1, + numberOfResults: 6, + rowsPerPage: 3, + }, + }, + 'normalize creates a `meta` prop for pagination' + ); }); }); diff --git a/packages/data/tests/unit/serializers/facts/bard-test.ts b/packages/data/tests/unit/serializers/facts/bard-test.ts index 2061510d5..1aac65dba 100644 --- a/packages/data/tests/unit/serializers/facts/bard-test.ts +++ b/packages/data/tests/unit/serializers/facts/bard-test.ts @@ -33,7 +33,6 @@ module('Unit | Serializer | facts/bard', function (hooks) { pagination: { currentPage: 1, rowsPerPage: 0, - perPage: 10, numberOfResults: 0, }, }, diff --git a/packages/data/tests/unit/serializers/facts/elide-test.ts b/packages/data/tests/unit/serializers/facts/elide-test.ts index 1a8459d85..6aab2d6bf 100644 --- a/packages/data/tests/unit/serializers/facts/elide-test.ts +++ b/packages/data/tests/unit/serializers/facts/elide-test.ts @@ -4,6 +4,7 @@ import NaviFactSerializer from 'navi-data/serializers/facts/interface'; import { AsyncQueryResponse, QueryStatus, RequestV2 } from 'navi-data/adapters/facts/interface'; import NaviFactResponse from 'navi-data/models/navi-fact-response'; import { ExecutionResult, GraphQLError } from 'graphql'; +import { getPaginationFromPageInfo } from 'navi-data/serializers/facts/elide'; const Payload: AsyncQueryResponse = { asyncQuery: { @@ -17,8 +18,18 @@ const Payload: AsyncQueryResponse = { contentLength: 129, httpStatus: 200, recordCount: 2, - responseBody: - '{"data":{"tableA":{"edges":[{"node":{"col0":"202003", "col1":10}},{"node":{"col0":"202004", "col1":20}}]}}}', + responseBody: JSON.stringify({ + data: { + tableA: { + edges: [{ node: { col0: '202003', col1: 10 } }, { node: { col0: '202004', col1: 20 } }], + pageInfo: { + startCursor: '0', + endCursor: '2', + totalRecords: 2, + }, + }, + }, + }), }, }, }, @@ -53,7 +64,13 @@ module('Unit | Serializer | facts/elide', function (hooks) { assert.deepEqual( { rows, meta }, { - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 2, + rowsPerPage: 2, + }, + }, rows: [ { datestamp: '202003', @@ -128,4 +145,43 @@ module('Unit | Serializer | facts/elide', function (hooks) { '`extractError` populates error object correctly when given an elide error' ); }); + + test('getPaginationFromPageInfo', async function (assert) { + const page = ( + startCursor: `${number}`, + endCursor: `${number}`, + totalRecords: number, + options?: Parameters[1] + ) => getPaginationFromPageInfo({ startCursor, endCursor, totalRecords }, options); + + assert.deepEqual( + page('0', '100', 100), + { + currentPage: 1, + rowsPerPage: 100, + numberOfResults: 100, + }, + 'A page with no offset works correctly' + ); + + assert.deepEqual( + page('100', '200', 1000), + { + currentPage: 2, + rowsPerPage: 100, + numberOfResults: 1000, + }, + 'A page with 1 offset works correctly' + ); + + assert.deepEqual( + page('900', '902', 902, { perPage: 100 }), + { + currentPage: 10, + rowsPerPage: 100, + numberOfResults: 902, + }, + 'Getting the last page, but passing in the perPage property shows the correct rowsPerPage' + ); + }); }); diff --git a/packages/data/tests/unit/services/navi-dimension-test.ts b/packages/data/tests/unit/services/navi-dimension-test.ts index ade474743..51d386fac 100644 --- a/packages/data/tests/unit/services/navi-dimension-test.ts +++ b/packages/data/tests/unit/services/navi-dimension-test.ts @@ -1,16 +1,26 @@ import { module, test } from 'qunit'; import { setupTest } from 'ember-qunit'; -import { DimensionFilter } from 'navi-data/adapters/dimensions/interface'; -import { TestContext as Context } from 'ember-test-helpers'; import NaviDimensionModel from 'navi-data/models/navi-dimension'; -import NaviDimensionService from 'navi-data/services/navi-dimension'; -import NaviMetadataService from 'navi-data/services/navi-metadata'; -import DimensionMetadataModel from 'navi-data/models/metadata/dimension'; // @ts-ignore import { setupMirage } from 'ember-cli-mirage/test-support'; import GraphQLScenario from 'navi-data/mirage/scenarios/elide-one'; -import { Server } from 'miragejs'; +import { task } from 'ember-concurrency'; import { taskFor } from 'ember-concurrency-ts'; +import config from 'ember-get-config'; +import EmberObject from '@ember/object'; +import DimensionMetadataModel, { DimensionColumn } from 'navi-data/models/metadata/dimension'; +import NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; +import type { TaskGenerator } from 'ember-concurrency'; +import type NaviDimensionAdapter from 'navi-data/adapters/dimensions/interface'; +import type { TestContext as Context } from 'ember-test-helpers'; +import type { DimensionFilter } from 'navi-data/adapters/dimensions/interface'; +import type NaviDimensionService from 'navi-data/services/navi-dimension'; +import type { ServiceOptions } from 'navi-data/services/navi-dimension'; +import type NaviMetadataService from 'navi-data/services/navi-metadata'; +import type { Server } from 'miragejs'; +import type { AsyncQueryResponse } from 'navi-data/adapters/facts/interface'; +import type NaviDimensionSerializer from 'navi-data/serializers/dimensions/interface'; +import { ResponseV1 } from 'navi-data/serializers/facts/interface'; interface TestContext extends Context { metadataService: NaviMetadataService; @@ -35,14 +45,184 @@ module('Unit | Service | navi-dimension', function (hooks) { 'elideOne' ) as DimensionMetadataModel; const expectedDimensionModels = [ - 'Handcrafted Frozen Mouse', - 'Licensed Soft Ball', 'Awesome Concrete Table', 'Handcrafted Concrete Mouse', + 'Handcrafted Frozen Mouse', + 'Licensed Soft Ball', ].map((dimVal) => NaviDimensionModel.create({ value: dimVal, dimensionColumn: { columnMetadata } })); const all = await taskFor(service.all).perform({ columnMetadata }); - assert.deepEqual(all, expectedDimensionModels, '`all` gets all the unfiltered values for a dimension'); + assert.deepEqual(all.values, expectedDimensionModels, '`all` gets all the unfiltered values for a dimension'); + }); + + test('all - pagination - generic', async function (this: TestContext, assert) { + assert.expect(15); + let originalDataSources = config.navi.dataSources; + const dataSourceType = 'mock'; + const dataSourceName = 'test-example'; + config.navi.dataSources = [{ type: dataSourceType, uri: 'fake', name: dataSourceName }]; + + let call = 0; + let adapterCallback = (_call: number, _options: ServiceOptions): void => undefined; + class MockAdapter extends EmberObject implements Pick { + @task *all(_dimension: DimensionColumn, options: ServiceOptions): TaskGenerator { + adapterCallback(call++, options); + return yield Promise.resolve({}); + } + } + this.owner.register(`adapter:dimensions/${dataSourceType}`, MockAdapter); + + let serializerCallback = (_call: number, _options: ServiceOptions): ResponseV1['meta'] => ({}); + class MockSerializer extends EmberObject implements NaviDimensionSerializer { + normalize(_dimension: DimensionColumn, _rawPayload: unknown, options: ServiceOptions): NaviDimensionResponse { + const meta = serializerCallback(call++, options); + return NaviDimensionResponse.create({ meta: meta }); + } + } + this.owner.register(`serializer:dimensions/${dataSourceType}`, MockSerializer); + + const service = this.owner.lookup('service:navi-dimension') as NaviDimensionService; + const columnMetadata = { source: dataSourceName } as DimensionMetadataModel; + + adapterCallback = (call, options) => { + if (call === 0) { + assert.deepEqual(options, {}, 'No options are passed on first call'); + } else if (call === 2) { + assert.deepEqual( + options, + { page: 2, perPage: 9 }, + 'The service uses the pagination options returned from the fetch to get the next page' + ); + } else if (call === 4) { + assert.deepEqual( + options, + { page: 3, perPage: 9 }, + 'The service uses the pagination options returned from the fetch until all pages are fetched' + ); + } else { + throw new Error('Adapter callback missed'); + } + }; + serializerCallback = (call, options) => { + if (call === 1) { + assert.deepEqual(options, {}, 'No options are passed on first call'); + return { pagination: { currentPage: 1, rowsPerPage: 9, numberOfResults: 20 } }; + } else if (call === 3) { + assert.deepEqual(options, { page: 2, perPage: 9 }, 'The pagination options are forwarded to the serializer'); + return { pagination: { currentPage: 2, rowsPerPage: 9, numberOfResults: 20 } }; + } else if (call === 5) { + assert.deepEqual(options, { page: 3, perPage: 9 }, 'The pagination options are forwarded to the serializer'); + return {}; + } else { + throw new Error('Serializer callback missed'); + } + }; + call = 0; + await taskFor(service.all).perform({ columnMetadata }); + assert.strictEqual(call, 6, 'It took 3 calls to adapter/serializer to page through all the data'); + + adapterCallback = (call, options) => { + if (call === 0) { + assert.deepEqual(options, {}, 'No options are passed on first call'); + } else { + throw new Error('Adapter callback missed'); + } + }; + serializerCallback = (call, options) => { + if (call === 1) { + assert.deepEqual(options, {}, 'No options are passed on first call'); + return {}; + } else { + throw new Error('Serializer callback missed'); + } + }; + call = 0; + await taskFor(service.all).perform({ columnMetadata }); + assert.strictEqual( + call, + 2, + 'The service called adapter and serializer then stopped because no pagination options were returned' + ); + + adapterCallback = (call, options) => { + if (call === 0) { + assert.deepEqual(options, { page: 1, perPage: 4 }, 'Page and perPage are present'); + } else if (call === 2) { + assert.deepEqual(options, { page: 2, perPage: 4 }, 'The next page is fetched using original pageSize'); + } else { + throw new Error('Adapter callback missed'); + } + }; + serializerCallback = (call, options) => { + if (call === 1) { + assert.deepEqual(options, { page: 1, perPage: 4 }, 'Page and perPage are present'); + return { pagination: { currentPage: 1, rowsPerPage: 4, numberOfResults: 5 } }; + } else if (call === 3) { + assert.deepEqual(options, { page: 2, perPage: 4 }, 'The next page is fetched using original pageSize'); + return {}; + } else { + throw new Error('Serializer callback missed'); + } + }; + call = 0; + await taskFor(service.all).perform({ columnMetadata }, { perPage: 4, page: 1 }); + assert.strictEqual(call, 4, 'It took 2 calls to adapter/serializer to page through all the data'); + + config.navi.dataSources = originalDataSources; + }); + + test('all - pagination - elide', async function (this: TestContext, assert) { + const service = this.owner.lookup('service:navi-dimension') as NaviDimensionService; + const columnMetadata = this.metadataService.getById( + 'dimension', + 'table0.dimension0', + 'elideOne' + ) as DimensionMetadataModel; + const all = await taskFor(service.all).perform({ columnMetadata }); + + assert.strictEqual(all.values.length, 4, 'There are 4 results'); + assert.strictEqual( + //@ts-expect-error -- the type does not expect a length property + this.server.db.asyncQueries.length, + 1, + 'Only one asyncQueries is created because response fits in one page' + ); + this.server.db.asyncQueries.remove(); + + const allPageBy1 = await taskFor(service.all).perform({ columnMetadata }, { perPage: 1 }); + assert.strictEqual( + //@ts-expect-error -- the type does not expect a length property + this.server.db.asyncQueries.length, + 4, + 'Four asyncQueries are created because the page size is 1 and there are 4 results' + ); + assert.deepEqual(allPageBy1.values, all.values, '`all` with paging gets the same values as without'); + }); + + test('all - pagination - fili', async function (this: TestContext, assert) { + const service = this.owner.lookup('service:navi-dimension') as NaviDimensionService; + await this.metadataService.loadMetadata({ dataSourceName: 'bardOne' }); + const columnMetadata = this.metadataService.getById('dimension', 'age', 'bardOne') as DimensionMetadataModel; + + let dataRequestsCount = 0; + this.server.pretender.handledRequest = (_, url) => { + if (url.includes('/v1/dimensions')) { + dataRequestsCount++; + } + }; + const all = await taskFor(service.all).perform({ columnMetadata }); + + assert.strictEqual(all.values.length, 13, 'There are 13 results'); + assert.strictEqual(dataRequestsCount, 1, 'Only one dimension request is created because response fits in one page'); + + dataRequestsCount = 0; + const allPageBy2 = await taskFor(service.all).perform({ columnMetadata }, { perPage: 1, page: 1 }); + assert.strictEqual(allPageBy2.values.length, 13, 'There are 13 results'); + assert.strictEqual( + dataRequestsCount, + 13, + '13 dimension requests are created because the page size is 1 and there are 13 results' + ); }); test('all - enum', async function (this: TestContext, assert) { @@ -62,7 +242,7 @@ module('Unit | Service | navi-dimension', function (hooks) { const all = await taskFor(service.all).perform({ columnMetadata }); assert.deepEqual( - all, + all.values, expectedDimensionModels, '`all` gets all the unfiltered values for a dimension with enum values' ); @@ -82,7 +262,7 @@ module('Unit | Service | navi-dimension', function (hooks) { ); const find = await taskFor(service.find).perform({ columnMetadata }, filters); assert.deepEqual( - find, + find.values, expectedDimensionModels, '`find` gets all the values for a dimension that match the specified filter' ); @@ -98,16 +278,16 @@ module('Unit | Service | navi-dimension', function (hooks) { 'elideOne' ) as DimensionMetadataModel; const search = await taskFor(service.search).perform({ columnMetadata }, 'plastic'); - const expectedDimensionModels = ['Licensed Plastic Pants', 'Awesome Plastic Fish'].map((dimVal) => + const expectedDimensionModels = ['Awesome Plastic Fish', 'Licensed Plastic Pants'].map((dimVal) => NaviDimensionModel.create({ value: dimVal, dimensionColumn: { columnMetadata } }) ); assert.deepEqual( - search, + search.values, expectedDimensionModels, '`search` gets all the values for a dimension that contain the query case insensitively' ); const noResultSearch = await taskFor(service.search).perform({ columnMetadata }, 'fuggedaboutit'); - assert.deepEqual(noResultSearch, [], 'Empty array is returned when no values are found'); + assert.deepEqual(noResultSearch.values, [], 'Empty array is returned when no values are found'); }); }); diff --git a/packages/data/tests/unit/services/navi-facts-elide-test.ts b/packages/data/tests/unit/services/navi-facts-elide-test.ts index a6fcca863..4e2c36a0f 100644 --- a/packages/data/tests/unit/services/navi-facts-elide-test.ts +++ b/packages/data/tests/unit/services/navi-facts-elide-test.ts @@ -32,14 +32,14 @@ const TestRequest: RequestV2 = { { field: 'table1.dimension2', operator: 'eq', - values: ['Incredible Metal Towels'], + values: ['Intelligent Soft Keyboard'], parameters: {}, type: 'dimension', }, { field: 'table1.dimension3', operator: 'notin', - values: ['Unbranded Soft Sausage', 'Ergonomic Plastic Tuna'], + values: ['Gorgeous Plastic Mouse', 'Unbranded Wooden Chair'], parameters: {}, type: 'dimension', }, @@ -101,146 +101,190 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { assert.deepEqual( { rows, meta }, { - meta: {}, - rows: [ - { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-01-29', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '231.96', - 'table1.metric2': '969.93', - 'table1.orderTimeDay': '2014-01-05', - }, - { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-01-29', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '236.73', - 'table1.metric2': '730.45', - 'table1.orderTimeDay': '2014-01-06', + meta: { + pagination: { + currentPage: 1, + numberOfResults: 12, + rowsPerPage: 12, }, + }, + rows: [ { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', 'table1.eventTimeDay': '2015-01-29', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '385.95', - 'table1.metric2': '463.94', + 'table1.metric1': '968.62', + 'table1.metric2': '550.28', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', 'table1.eventTimeDay': '2015-01-29', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '998.39', - 'table1.metric2': '433.80', + 'table1.metric1': '890.81', + 'table1.metric2': '182.06', 'table1.orderTimeDay': '2014-01-06', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', 'table1.eventTimeDay': '2015-01-30', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '389.34', - 'table1.metric2': '661.33', + 'table1.metric1': '362.00', + 'table1.metric2': '2.19', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', 'table1.eventTimeDay': '2015-01-30', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '451.75', - 'table1.metric2': '355.84', + 'table1.metric1': '272.86', + 'table1.metric2': '169.29', 'table1.orderTimeDay': '2014-01-06', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', - 'table1.eventTimeDay': '2015-01-30', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-01-31', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '723.84', - 'table1.metric2': '196.83', + 'table1.metric1': '774.54', + 'table1.metric2': '146.85', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', - 'table1.eventTimeDay': '2015-01-30', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-01-31', 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '476.87', - 'table1.metric2': '676.99', + 'table1.metric1': '317.51', + 'table1.metric2': '943.86', 'table1.orderTimeDay': '2014-01-06', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-01-31', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '545.26', - 'table1.metric2': '114.62', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-01', + 'table1.eventTimeMonth': '2015 Feb', + 'table1.metric1': '987.85', + 'table1.metric2': '583.42', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-01-31', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '589.71', - 'table1.metric2': '496.48', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-01', + 'table1.eventTimeMonth': '2015 Feb', + 'table1.metric1': '708.99', + 'table1.metric2': '417.93', 'table1.orderTimeDay': '2014-01-06', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', - 'table1.eventTimeDay': '2015-01-31', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '432.79', - 'table1.metric2': '246.23', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-02', + 'table1.eventTimeMonth': '2015 Feb', + 'table1.metric1': '498.34', + 'table1.metric2': '971.55', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', - 'table1.eventTimeDay': '2015-01-31', - 'table1.eventTimeMonth': '2015 Jan', - 'table1.metric1': '104.97', - 'table1.metric2': '682.74', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-02', + 'table1.eventTimeMonth': '2015 Feb', + 'table1.metric1': '145.24', + 'table1.metric2': '335.43', 'table1.orderTimeDay': '2014-01-06', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-02-01', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-03', 'table1.eventTimeMonth': '2015 Feb', - 'table1.metric1': '861.11', - 'table1.metric2': '824.58', + 'table1.metric1': '508.71', + 'table1.metric2': '510.03', 'table1.orderTimeDay': '2014-01-05', }, { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Unbranded Soft Sausages', - 'table1.eventTimeDay': '2015-02-01', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-02-03', 'table1.eventTimeMonth': '2015 Feb', - 'table1.metric1': '486.71', - 'table1.metric2': '482.62', + 'table1.metric1': '316.59', + 'table1.metric2': '437.07', 'table1.orderTimeDay': '2014-01-06', }, + ], + }, + 'Request V2 query is properly sent with all necessary arguments supplied' + ); + }); + + test('fetch - pagination', async function (this: TestContext, assert) { + const limitless = { ...TestRequest, limit: null }; + const model = await taskFor(this.service.fetch).perform(limitless, { + dataSourceName: limitless.dataSource, + page: 1, + perPage: 1, + }); + const { rows, meta } = model.response as NaviFactResponse; + assert.deepEqual( + { rows, meta }, + { + meta: { + pagination: { + currentPage: 1, + numberOfResults: 12, + rowsPerPage: 1, + }, + }, + rows: [ { - 'table1.dimension2': 'Incredible Metal Towels', - 'table1.dimension3': 'Ergonomic Steel Sausages', - 'table1.eventTimeDay': '2015-02-01', - 'table1.eventTimeMonth': '2015 Feb', - 'table1.metric1': '308.03', - 'table1.metric2': '227.94', + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-01-29', + 'table1.eventTimeMonth': '2015 Jan', + 'table1.metric1': '968.62', + 'table1.metric2': '550.28', 'table1.orderTimeDay': '2014-01-05', }, ], }, - 'Request V2 query is properly sent with all necessary arguments supplied' + 'The pagination options limit the response to 1 row and gets the first page' + ); + + const modelPage2 = await taskFor(this.service.fetch).perform(limitless, { + dataSourceName: limitless.dataSource, + page: 2, + perPage: 1, + }); + const { rows: rows2, meta: meta2 } = modelPage2.response as NaviFactResponse; + assert.deepEqual( + { rows: rows2, meta: meta2 }, + { + meta: { + pagination: { + currentPage: 2, + numberOfResults: 12, + rowsPerPage: 1, + }, + }, + rows: [ + { + 'table1.dimension2': 'Intelligent Soft Keyboard', + 'table1.dimension3': 'Ergonomic Plastic Shirt', + 'table1.eventTimeDay': '2015-01-29', + 'table1.eventTimeMonth': '2015 Jan', + 'table1.metric1': '890.81', + 'table1.metric2': '182.06', + 'table1.orderTimeDay': '2014-01-06', + }, + ], + }, + 'The pagination options limit the response to 1 row and gets the second page' ); }); @@ -264,7 +308,16 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { const { rows, meta } = model.response as NaviFactResponse; assert.deepEqual( { rows, meta }, - { rows: [{ 'table1.metric1': '823.11', 'table1.metric2': '823.38' }], meta: {} }, + { + meta: { + pagination: { + currentPage: 1, + numberOfResults: 1, + rowsPerPage: 1, + }, + }, + rows: [{ 'table1.metric1': '250.92', 'table1.metric2': '680.57' }], + }, 'Request with only metrics is formatted correctly' ); }); @@ -321,8 +374,14 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { assert.deepEqual( { rows, meta }, { + meta: { + pagination: { + currentPage: 1, + numberOfResults: 0, + rowsPerPage: 0, + }, + }, rows: [], - meta: {}, }, 'An invalid filter on a requested field returns an empty response' ); @@ -347,9 +406,16 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { rows: noTimeDimResponse?.rows, meta: noTimeDimResponse?.meta, }, + { rows: [{ 'table1.metric1': '307.93' }], - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 1, + rowsPerPage: 1, + }, + }, }, 'An invalid filter on a non-requested field does not affect the response' ); @@ -388,7 +454,13 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { { 'table1.eventTimeMonth': '2015 Jan', 'table1.metric1': '17.49' }, { 'table1.eventTimeMonth': '2015 Feb', 'table1.metric1': '426.48' }, ], - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 2, + rowsPerPage: 2, + }, + }, }, 'A date filter with no end date defaults to a one month date interval' ); @@ -428,7 +500,13 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { { 'table1.eventTimeMonth': '2014 Nov', 'table1.metric1': '17.49' }, { 'table1.eventTimeMonth': '2014 Dec', 'table1.metric1': '426.48' }, ], - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 2, + rowsPerPage: 2, + }, + }, }, 'A date filter with no end date defaults to a one month date interval' ); @@ -474,7 +552,13 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { 'table1.eventTimeDay': moment().utc().format(DAY_FORMAT), }, ], - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 3, + rowsPerPage: 3, + }, + }, }, 'A date filter with no end date ends at current if start is not more than a month before current' ); @@ -516,19 +600,25 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { assert.deepEqual( { rows, meta }, { - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 3, + rowsPerPage: 3, + }, + }, rows: [ { - 'table1.eventTimeDay': '2015-01-03', - 'table1.metric1': '44.71', + 'table1.eventTimeDay': '2015-01-02', + 'table1.metric1': '258.04', }, { - 'table1.eventTimeDay': '2015-01-02', - 'table1.metric1': '327.11', + 'table1.eventTimeDay': '2015-01-03', + 'table1.metric1': '634.84', }, { 'table1.eventTimeDay': '2015-01-01', - 'table1.metric1': '675.73', + 'table1.metric1': '783.84', }, ], }, @@ -546,8 +636,8 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { ], filters: [], sorts: [ - { field: 'table1.dimension2', parameters: {}, type: 'metric', direction: 'asc' }, - { field: 'table1.dimension3', parameters: {}, type: 'metric', direction: 'asc' }, + { type: 'dimension', field: 'table1.dimension2', parameters: {}, direction: 'asc' }, + { type: 'dimension', field: 'table1.dimension3', parameters: {}, direction: 'asc' }, ], limit: null, requestVersion: '2.0', @@ -563,67 +653,138 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { meta: multiSortResponse?.meta, }, { - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 25, + rowsPerPage: 25, + }, + }, rows: [ { - 'table1.dimension2': 'Handmade Rubber Fish', - 'table1.dimension3': 'Awesome Cotton Sausages', - 'table1.metric1': '913.34', + 'table1.dimension2': 'Ergonomic Plastic Bacon', + 'table1.dimension3': 'Awesome Cotton Bacon', + 'table1.metric1': '849.74', }, { - 'table1.dimension2': 'Handmade Rubber Fish', - 'table1.dimension3': 'Handcrafted Concrete Pizza', - 'table1.metric1': '220.58', + 'table1.dimension2': 'Ergonomic Plastic Bacon', + 'table1.dimension3': 'Ergonomic Steel Tuna', + 'table1.metric1': '651.12', }, { - 'table1.dimension2': 'Handmade Rubber Fish', - 'table1.dimension3': 'Small Metal Mouse', - 'table1.metric1': '286.62', + 'table1.dimension2': 'Ergonomic Plastic Bacon', + 'table1.dimension3': 'Handcrafted Rubber Pizza', + 'table1.metric1': '440.87', }, { - 'table1.dimension2': 'Handmade Rubber Fish', - 'table1.dimension3': 'Unbranded Wooden Pizza', - 'table1.metric1': '188.92', + 'table1.dimension2': 'Ergonomic Plastic Bacon', + 'table1.dimension3': 'Incredible Metal Computer', + 'table1.metric1': '922.05', }, { - 'table1.dimension2': 'Incredible Rubber Tuna', - 'table1.dimension3': 'Awesome Cotton Sausages', - 'table1.metric1': '403.35', + 'table1.dimension2': 'Ergonomic Plastic Bacon', + 'table1.dimension3': 'Small Fresh Car', + 'table1.metric1': '265.31', }, { - 'table1.dimension2': 'Incredible Rubber Tuna', - 'table1.dimension3': 'Handcrafted Concrete Pizza', - 'table1.metric1': '500.33', + 'table1.dimension2': 'Generic Plastic Mouse', + 'table1.dimension3': 'Awesome Cotton Bacon', + 'table1.metric1': '712.64', }, { - 'table1.dimension2': 'Incredible Rubber Tuna', - 'table1.dimension3': 'Small Metal Mouse', - 'table1.metric1': '131.54', + 'table1.dimension2': 'Generic Plastic Mouse', + 'table1.dimension3': 'Ergonomic Steel Tuna', + 'table1.metric1': '624.14', }, { - 'table1.dimension2': 'Incredible Rubber Tuna', - 'table1.dimension3': 'Unbranded Wooden Pizza', - 'table1.metric1': '441.55', + 'table1.dimension2': 'Generic Plastic Mouse', + 'table1.dimension3': 'Handcrafted Rubber Pizza', + 'table1.metric1': '19.73', }, { - 'table1.dimension2': 'Licensed Concrete Fish', - 'table1.dimension3': 'Awesome Cotton Sausages', - 'table1.metric1': '528.84', + 'table1.dimension2': 'Generic Plastic Mouse', + 'table1.dimension3': 'Incredible Metal Computer', + 'table1.metric1': '701.89', }, { - 'table1.dimension2': 'Licensed Concrete Fish', - 'table1.dimension3': 'Handcrafted Concrete Pizza', - 'table1.metric1': '992.46', + 'table1.dimension2': 'Generic Plastic Mouse', + 'table1.dimension3': 'Small Fresh Car', + 'table1.metric1': '35.67', }, { - 'table1.dimension2': 'Licensed Concrete Fish', - 'table1.dimension3': 'Small Metal Mouse', - 'table1.metric1': '337.40', + 'table1.dimension2': 'Handcrafted Cotton Mouse', + 'table1.dimension3': 'Awesome Cotton Bacon', + 'table1.metric1': '391.73', }, { - 'table1.dimension2': 'Licensed Concrete Fish', - 'table1.dimension3': 'Unbranded Wooden Pizza', - 'table1.metric1': '974.83', + 'table1.dimension2': 'Handcrafted Cotton Mouse', + 'table1.dimension3': 'Ergonomic Steel Tuna', + 'table1.metric1': '736.36', + }, + { + 'table1.dimension2': 'Handcrafted Cotton Mouse', + 'table1.dimension3': 'Handcrafted Rubber Pizza', + 'table1.metric1': '154.11', + }, + { + 'table1.dimension2': 'Handcrafted Cotton Mouse', + 'table1.dimension3': 'Incredible Metal Computer', + 'table1.metric1': '880.89', + }, + { + 'table1.dimension2': 'Handcrafted Cotton Mouse', + 'table1.dimension3': 'Small Fresh Car', + 'table1.metric1': '666.88', + }, + { + 'table1.dimension2': 'Licensed Concrete Bike', + 'table1.dimension3': 'Awesome Cotton Bacon', + 'table1.metric1': '719.31', + }, + { + 'table1.dimension2': 'Licensed Concrete Bike', + 'table1.dimension3': 'Ergonomic Steel Tuna', + 'table1.metric1': '563.02', + }, + { + 'table1.dimension2': 'Licensed Concrete Bike', + 'table1.dimension3': 'Handcrafted Rubber Pizza', + 'table1.metric1': '356.45', + }, + { + 'table1.dimension2': 'Licensed Concrete Bike', + 'table1.dimension3': 'Incredible Metal Computer', + 'table1.metric1': '327.80', + }, + { + 'table1.dimension2': 'Licensed Concrete Bike', + 'table1.dimension3': 'Small Fresh Car', + 'table1.metric1': '303.88', + }, + { + 'table1.dimension2': 'Tasty Granite Computer', + 'table1.dimension3': 'Awesome Cotton Bacon', + 'table1.metric1': '745.72', + }, + { + 'table1.dimension2': 'Tasty Granite Computer', + 'table1.dimension3': 'Ergonomic Steel Tuna', + 'table1.metric1': '324.94', + }, + { + 'table1.dimension2': 'Tasty Granite Computer', + 'table1.dimension3': 'Handcrafted Rubber Pizza', + 'table1.metric1': '990.20', + }, + { + 'table1.dimension2': 'Tasty Granite Computer', + 'table1.dimension3': 'Incredible Metal Computer', + 'table1.metric1': '796.04', + }, + { + 'table1.dimension2': 'Tasty Granite Computer', + 'table1.dimension3': 'Small Fresh Car', + 'table1.metric1': '729.57', }, ], }, @@ -671,19 +832,25 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { meta: limit?.meta, }, { - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 9, + rowsPerPage: 3, + }, + }, rows: [ { 'table1.eventTimeDay': '2015-01-01', - 'table1.metric1': '823.11', + 'table1.metric1': '783.84', }, { 'table1.eventTimeDay': '2015-01-02', - 'table1.metric1': '823.38', + 'table1.metric1': '258.04', }, { 'table1.eventTimeDay': '2015-01-03', - 'table1.metric1': '26.11', + 'table1.metric1': '634.84', }, ], }, @@ -729,7 +896,13 @@ module('Unit | Service | Navi Facts - Elide', function (hooks) { meta: limitless?.meta, }, { - meta: {}, + meta: { + pagination: { + currentPage: 1, + numberOfResults: 9, + rowsPerPage: 9, + }, + }, rows: [ { 'table1.eventTimeDay': '2015-01-01', diff --git a/packages/data/tests/unit/services/navi-facts-test.ts b/packages/data/tests/unit/services/navi-facts-test.ts index a708c29aa..c2ec2fc68 100644 --- a/packages/data/tests/unit/services/navi-facts-test.ts +++ b/packages/data/tests/unit/services/navi-facts-test.ts @@ -293,22 +293,20 @@ module('Unit | Service | Navi Facts', function (hooks) { }); const service: NaviFactsService = this.owner.lookup('service:navi-facts'); - const response: ResponseV1 = { + const response = NaviFactResponse.create({ rows: [], meta: { - //@ts-expect-error pagination: { currentPage: 2, - perPage: 10, + rowsPerPage: 10, numberOfResults: 30, }, }, - }; + }); const request = {} as RequestV2; await taskFor(service.fetchNext).perform(response, request); - //@ts-expect-error response.meta.pagination.currentPage = 3; assert.equal( await taskFor(service.fetchNext).perform(response, request), @@ -325,22 +323,20 @@ module('Unit | Service | Navi Facts', function (hooks) { }); const service: NaviFactsService = this.owner.lookup('service:navi-facts'); - const response: ResponseV1 = { + const response = NaviFactResponse.create({ rows: [], meta: { - //@ts-expect-error pagination: { currentPage: 2, - perPage: 10, + rowsPerPage: 10, numberOfResults: 30, }, }, - }; + }); const request = {} as RequestV2; await taskFor(service.fetchPrevious).perform(response, request); - //@ts-expect-error response.meta.pagination.currentPage = 1; assert.equal( await taskFor(service.fetchPrevious).perform(response, request), diff --git a/packages/reports/addon/components/filter-values/dimension-select.ts b/packages/reports/addon/components/filter-values/dimension-select.ts index 46e8562cb..7644e7d22 100644 --- a/packages/reports/addon/components/filter-values/dimension-select.ts +++ b/packages/reports/addon/components/filter-values/dimension-select.ts @@ -10,6 +10,8 @@ import { tracked } from '@glimmer/tracking'; import { task, TaskGenerator, timeout } from 'ember-concurrency'; import CARDINALITY_SIZES from 'navi-data/utils/enums/cardinality-sizes'; import NaviDimensionModel from 'navi-data/models/navi-dimension'; +import { sortBy } from 'lodash-es'; +import { taskFor } from 'ember-concurrency-ts'; import type NaviDimensionService from 'navi-data/services/navi-dimension'; import type NaviMetadataService from 'navi-data/services/navi-metadata'; import type RequestFragment from 'navi-core/models/bard-request-v2/request'; @@ -17,8 +19,7 @@ import type FilterFragment from 'navi-core/models/bard-request-v2/fragments/filt import type DimensionMetadataModel from 'navi-data/models/metadata/dimension'; import type { DimensionColumn } from 'navi-data/models/metadata/dimension'; import type { IndexedOptions } from '../power-select-collection-options'; -import { sortBy } from 'lodash-es'; -import { taskFor } from 'ember-concurrency-ts'; +import type NaviDimensionResponse from 'navi-data/models/navi-dimension-response'; const SEARCH_DEBOUNCE_MS = 250; const SEARCH_DEBOUNCE_OFFLINE_MS = 100; @@ -94,7 +95,9 @@ export default class DimensionSelectComponent extends Component r.values); } } } @@ -113,7 +116,11 @@ export default class DimensionSelectComponent extends Component { - const values = ['1', '3', '2', '11', '111']; - return yield values.map((value) => NaviDimensionModel.create({ value, dimensionColumn })); + const rawValues = ['1', '3', '2', '11', '111']; + const values = rawValues.map((value) => NaviDimensionModel.create({ value, dimensionColumn })); + return yield NaviDimensionResponse.create({ values }); } } @@ -241,8 +243,9 @@ module('Integration | Component | filter values/dimension select', function (hoo class MockDimensions extends Service { @task *all(dimensionColumn: DimensionColumn): TaskGenerator { - const values = ['1', '3', '2', '11', '111', 'stringvalue']; - return yield values.map((value) => NaviDimensionModel.create({ value, dimensionColumn })); + const rawValues = ['1', '3', '2', '11', '111', 'stringvalue']; + const values = rawValues.map((value) => NaviDimensionModel.create({ value, dimensionColumn })); + return yield NaviDimensionResponse.create({ values }); } } @@ -264,8 +267,11 @@ module('Integration | Component | filter values/dimension select', function (hoo class MockDimensions extends Service { @task *search(dimensionColumn: DimensionColumn): TaskGenerator { - const values = ['1', '3', '2', '11', '111']; - return yield values.map((value) => NaviDimensionModel.create({ value: `Property ${value}`, dimensionColumn })); + const rawValues = ['1', '3', '2', '11', '111']; + const values = rawValues.map((value) => + NaviDimensionModel.create({ value: `Property ${value}`, dimensionColumn }) + ); + return yield NaviDimensionResponse.create({ values }); } }