diff --git a/packages/lu/src/parser/cross-train/cross-train.js b/packages/lu/src/parser/cross-train/cross-train.js index 421a02698..67c621c6b 100644 --- a/packages/lu/src/parser/cross-train/cross-train.js +++ b/packages/lu/src/parser/cross-train/cross-train.js @@ -4,7 +4,8 @@ */ const path = require('path') -const file = require('../../utils/filehelper') +const fs = require('fs-extra') +const filehelper = require('../../utils/filehelper') const fileExtEnum = require('../utils/helpers').FileExtTypeEnum const crossTrainer = require('./crossTrainer') @@ -20,26 +21,43 @@ module.exports = { */ train: async function (input, intentName, config, verbose, trainingOpt) { // Get all related file content. - const luContents = await file.getFilesContent(input, fileExtEnum.LUFile) - const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile) - const configContent = await file.getConfigContent(config) + const luContents = await filehelper.getFilesContent(input, fileExtEnum.LUFile) + const qnaContents = await filehelper.getFilesContent(input, fileExtEnum.QnAFile) + const configContent = await filehelper.getConfigContent(config) - let importResolver = async function (_, idsToFind) { + let importResolver = async function (id, idsToFind) { let importedContents = [] for (let idx = 0; idx < idsToFind.length; idx++) { let file = idsToFind[idx] if (path.isAbsolute(file.filePath)) { if (file.filePath.endsWith(fileExtEnum.LUFile)) { - importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.LUFile)) + importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile)) } else if (file.filePath.endsWith(fileExtEnum.QnAFile)) { - importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.QnAFile)) + importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.QnAFile)) } } else { const fileName = path.basename(file.filePath) + const updateImportedContents = async function(typedContents, fileExt) { + const found = typedContents.filter(content => content.id === path.basename(fileName, fileExt)) + if(found.length > 0) { + importedContents.push(...found) + } else { + const matchedLuisFiles = typedContents.filter(content => path.basename(content.fullPath) === id) + for (const matchFile of matchedLuisFiles) { + const sourceFileDir = path.dirname(matchFile.fullPath) + const targetPath = path.resolve(sourceFileDir, file.filePath) + if (fs.existsSync(targetPath)) { + const importContent = await filehelper.getFilesContent(targetPath, fileExt) + importedContents.push(...importContent) + } + } + } + } + if (fileName.endsWith(fileExtEnum.LUFile)) { - importedContents.push(...luContents.filter(luContent => luContent.id === path.basename(fileName, fileExtEnum.LUFile))) + await updateImportedContents(luContents, fileExtEnum.LUFile) } else if (fileName.endsWith(fileExtEnum.QnAFile)) { - importedContents.push(...qnaContents.filter(qnaContent => qnaContent.id === path.basename(fileName, fileExtEnum.QnAFile))) + await updateImportedContents(qnaContents, fileExtEnum.LUFile) } } } @@ -60,4 +78,4 @@ module.exports = { return trainedResult } -} +} \ No newline at end of file diff --git a/packages/lu/src/utils/filehelper.ts b/packages/lu/src/utils/filehelper.ts index 32eecc393..77329dc66 100644 --- a/packages/lu/src/utils/filehelper.ts +++ b/packages/lu/src/utils/filehelper.ts @@ -176,7 +176,7 @@ export async function getFilesContent(input: string, extType: string) { if (fileStat.isFile()) { const filePath = path.resolve(input) const content = await getContentFromFile(input) - return [{id: path.basename(filePath, extType), content}] + return [{id: path.basename(filePath, extType), content, fullPath: filePath}] } if (!fileStat.isDirectory()) { @@ -186,7 +186,7 @@ export async function getFilesContent(input: string, extType: string) { return Promise.all(paths.map(async (item: string) => { const itemPath = path.resolve(path.join(input, item)) const content = await getContentFromFile(itemPath) - return {id: path.basename(itemPath, extType), content} + return {id: path.basename(itemPath, extType), content, fullPath: itemPath} })) } diff --git a/packages/luis/test/commands/luis/crossTrain.test.ts b/packages/luis/test/commands/luis/crossTrain.test.ts index 310091d33..5de70bc2c 100644 --- a/packages/luis/test/commands/luis/crossTrain.test.ts +++ b/packages/luis/test/commands/luis/crossTrain.test.ts @@ -148,4 +148,16 @@ describe('luis:cross-train tests for lu and qna contents', () => { expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption7/dia1.lu')).to.be.true expect(await compareLuFiles('./../../../interruptionGen/dia1.qna', './../../fixtures/verified/interruption7/dia1.qna')).to.be.true }) + + test + .stdout() + .command(['luis:cross-train', + '--in', `${path.join(__dirname, './../../fixtures/testcases/application')}`, + '--config', `${path.join(__dirname, './../../fixtures/testcases/application/cross-train.config')}`, + '--out', './interruptionGen', + '--force']) + .it('luis:cross training should able to import files out of current directory', async () => { + expect(await compareLuFiles('./../../../interruptionGen/application.lu', './../../fixtures/verified/interruption8/application.lu')).to.be.true + expect(await compareLuFiles('./../../../interruptionGen/application.qna', './../../fixtures/verified/interruption8/application.qna')).to.be.true + }) }) \ No newline at end of file diff --git a/packages/luis/test/fixtures/testcases/LUIS/ExternalFile.lu b/packages/luis/test/fixtures/testcases/LUIS/ExternalFile.lu new file mode 100644 index 000000000..7d143027f --- /dev/null +++ b/packages/luis/test/fixtures/testcases/LUIS/ExternalFile.lu @@ -0,0 +1,2 @@ +# ExternalIntent +- utterance for ExternalIntent \ No newline at end of file diff --git a/packages/luis/test/fixtures/testcases/QnAMaker/ExternalFile.qna b/packages/luis/test/fixtures/testcases/QnAMaker/ExternalFile.qna new file mode 100644 index 000000000..ecf98c547 --- /dev/null +++ b/packages/luis/test/fixtures/testcases/QnAMaker/ExternalFile.qna @@ -0,0 +1,5 @@ +# ? Question In ExternalFile.qna + +``` +Answer to question in ExternalFile.qna +``` \ No newline at end of file diff --git a/packages/luis/test/fixtures/testcases/application/LUIS/application.lu b/packages/luis/test/fixtures/testcases/application/LUIS/application.lu new file mode 100644 index 000000000..369008e1c --- /dev/null +++ b/packages/luis/test/fixtures/testcases/application/LUIS/application.lu @@ -0,0 +1,4 @@ +[ExternalFile](../../LUIS/ExternalFile.lu) + +# LocalIntent +- utterance for LocalIntent \ No newline at end of file diff --git a/packages/luis/test/fixtures/testcases/application/QnAMaker/application.qna b/packages/luis/test/fixtures/testcases/application/QnAMaker/application.qna new file mode 100644 index 000000000..8985c44db --- /dev/null +++ b/packages/luis/test/fixtures/testcases/application/QnAMaker/application.qna @@ -0,0 +1,7 @@ +[ExternalFile](../../QnaMaker/ExternalFile.qna) + +# ? Question In Main application.qna + +``` +Answer to question in application.qna +``` \ No newline at end of file diff --git a/packages/luis/test/fixtures/testcases/application/cross-train.config b/packages/luis/test/fixtures/testcases/application/cross-train.config new file mode 100644 index 000000000..2be218184 --- /dev/null +++ b/packages/luis/test/fixtures/testcases/application/cross-train.config @@ -0,0 +1,5 @@ +{ + "application": { + "rootDialog": true + } +} diff --git a/packages/luis/test/fixtures/verified/interruption8/application.lu b/packages/luis/test/fixtures/verified/interruption8/application.lu new file mode 100644 index 000000000..b2ae5f3f0 --- /dev/null +++ b/packages/luis/test/fixtures/verified/interruption8/application.lu @@ -0,0 +1,37 @@ + +> LUIS application information +> !# @app.versionId = 0.1 +> !# @app.culture = en-us +> !# @app.luis_schema_version = 3.2.0 + + +> # Intent definitions + +# LocalIntent +- utterance for LocalIntent + + +# ExternalIntent +- utterance for ExternalIntent + + +> # Entity definitions + + +> # PREBUILT Entity definitions + + +> # Phrase list definitions + + +> # List entities + +> # RegEx entities + + + + +> Source: cross training. Please do not edit these directly! +# DeferToRecognizer_QnA_application +- Question In Main application.qna +- Question In ExternalFile.qna \ No newline at end of file diff --git a/packages/luis/test/fixtures/verified/interruption8/application.qna b/packages/luis/test/fixtures/verified/interruption8/application.qna new file mode 100644 index 000000000..b3cf5a83f --- /dev/null +++ b/packages/luis/test/fixtures/verified/interruption8/application.qna @@ -0,0 +1,30 @@ +# ? Question In Main application.qna + +**Filters:** +- dialogName=application + +``` +Answer to question in application.qna +``` + +# ? Question In ExternalFile.qna + +**Filters:** +- dialogName=application + +``` +Answer to question in ExternalFile.qna +``` + +> Source: cross training. Please do not edit these directly! +> !# @qna.pair.source = crosstrained + +# ? utterance for LocalIntent +- utterance for ExternalIntent + +**Filters:** +- dialogName=application + +``` +intent=DeferToRecognizer_LUIS_application +``` \ No newline at end of file