Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.
Merged
38 changes: 28 additions & 10 deletions packages/lu/src/parser/cross-train/cross-train.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
*/

const path = require('path')
const file = require('../../utils/filehelper')
const fs = require('fs-extra')
const filehelper = require('../../utils/filehelper')
const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
const crossTrainer = require('./crossTrainer')

Expand All @@ -20,26 +21,43 @@ module.exports = {
*/
train: async function (input, intentName, config, verbose, trainingOpt) {
// Get all related file content.
const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
const configContent = await file.getConfigContent(config)
const luContents = await filehelper.getFilesContent(input, fileExtEnum.LUFile)
const qnaContents = await filehelper.getFilesContent(input, fileExtEnum.QnAFile)
const configContent = await filehelper.getConfigContent(config)

let importResolver = async function (_, idsToFind) {
let importResolver = async function (id, idsToFind) {
let importedContents = []
for (let idx = 0; idx < idsToFind.length; idx++) {
let file = idsToFind[idx]
if (path.isAbsolute(file.filePath)) {
if (file.filePath.endsWith(fileExtEnum.LUFile)) {
importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.LUFile))
importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile))
} else if (file.filePath.endsWith(fileExtEnum.QnAFile)) {
importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.QnAFile))
importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.QnAFile))
}
} else {
const fileName = path.basename(file.filePath)
const updateImportedContents = async function(typedContents, fileExt) {
const found = typedContents.filter(content => content.id === path.basename(fileName, fileExt))
if(found.length > 0) {
importedContents.push(...found)
} else {
const matchedLuisFiles = typedContents.filter(content => path.basename(content.fullPath) === id)
for (const matchFile of matchedLuisFiles) {
const sourceFileDir = path.dirname(matchFile.fullPath)
const targetPath = path.resolve(sourceFileDir, file.filePath)
if (fs.existsSync(targetPath)) {
const importContent = await filehelper.getFilesContent(targetPath, fileExt)
importedContents.push(...importContent)
}
}
}
}

if (fileName.endsWith(fileExtEnum.LUFile)) {
importedContents.push(...luContents.filter(luContent => luContent.id === path.basename(fileName, fileExtEnum.LUFile)))
await updateImportedContents(luContents, fileExtEnum.LUFile)
} else if (fileName.endsWith(fileExtEnum.QnAFile)) {
importedContents.push(...qnaContents.filter(qnaContent => qnaContent.id === path.basename(fileName, fileExtEnum.QnAFile)))
await updateImportedContents(qnaContents, fileExtEnum.LUFile)
}
}
}
Expand All @@ -60,4 +78,4 @@ module.exports = {

return trainedResult
}
}
}
4 changes: 2 additions & 2 deletions packages/lu/src/utils/filehelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ export async function getFilesContent(input: string, extType: string) {
if (fileStat.isFile()) {
const filePath = path.resolve(input)
const content = await getContentFromFile(input)
return [{id: path.basename(filePath, extType), content}]
return [{id: path.basename(filePath, extType), content, fullPath: filePath}]
}

if (!fileStat.isDirectory()) {
Expand All @@ -186,7 +186,7 @@ export async function getFilesContent(input: string, extType: string) {
return Promise.all(paths.map(async (item: string) => {
const itemPath = path.resolve(path.join(input, item))
const content = await getContentFromFile(itemPath)
return {id: path.basename(itemPath, extType), content}
return {id: path.basename(itemPath, extType), content, fullPath: itemPath}
}))
}

Expand Down
12 changes: 12 additions & 0 deletions packages/luis/test/commands/luis/crossTrain.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,16 @@ describe('luis:cross-train tests for lu and qna contents', () => {
expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption7/dia1.lu')).to.be.true
expect(await compareLuFiles('./../../../interruptionGen/dia1.qna', './../../fixtures/verified/interruption7/dia1.qna')).to.be.true
})

test
.stdout()
.command(['luis:cross-train',
'--in', `${path.join(__dirname, './../../fixtures/testcases/application')}`,
'--config', `${path.join(__dirname, './../../fixtures/testcases/application/cross-train.config')}`,
'--out', './interruptionGen',
'--force'])
.it('luis:cross training should able to import files out of current directory', async () => {
expect(await compareLuFiles('./../../../interruptionGen/application.lu', './../../fixtures/verified/interruption8/application.lu')).to.be.true
expect(await compareLuFiles('./../../../interruptionGen/application.qna', './../../fixtures/verified/interruption8/application.qna')).to.be.true
})
})
2 changes: 2 additions & 0 deletions packages/luis/test/fixtures/testcases/LUIS/ExternalFile.lu
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# ExternalIntent
- utterance for ExternalIntent
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# ? Question In ExternalFile.qna

```
Answer to question in ExternalFile.qna
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[ExternalFile](../../LUIS/ExternalFile.lu)

# LocalIntent
- utterance for LocalIntent
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[ExternalFile](../../QnaMaker/ExternalFile.qna)

# ? Question In Main application.qna

```
Answer to question in application.qna
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"application": {
"rootDialog": true
}
}
37 changes: 37 additions & 0 deletions packages/luis/test/fixtures/verified/interruption8/application.lu
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

> LUIS application information
> !# @app.versionId = 0.1
> !# @app.culture = en-us
> !# @app.luis_schema_version = 3.2.0


> # Intent definitions

# LocalIntent
- utterance for LocalIntent


# ExternalIntent
- utterance for ExternalIntent


> # Entity definitions


> # PREBUILT Entity definitions


> # Phrase list definitions


> # List entities

> # RegEx entities




> Source: cross training. Please do not edit these directly!
# DeferToRecognizer_QnA_application
- Question In Main application.qna
- Question In ExternalFile.qna
30 changes: 30 additions & 0 deletions packages/luis/test/fixtures/verified/interruption8/application.qna
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# ? Question In Main application.qna

**Filters:**
- dialogName=application

```
Answer to question in application.qna
```

# ? Question In ExternalFile.qna

**Filters:**
- dialogName=application

```
Answer to question in ExternalFile.qna
```

> Source: cross training. Please do not edit these directly!
> !# @qna.pair.source = crosstrained

# ? utterance for LocalIntent
- utterance for ExternalIntent

**Filters:**
- dialogName=application

```
intent=DeferToRecognizer_LUIS_application
```