diff --git a/.circleci/config.yml b/.circleci/config.yml index b09d83f1..17a2caca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,7 +68,7 @@ jobs: working_directory: ~/code-gov-api docker: - image: circleci/node:8.9.3-stretch - - image: elasticsearch:2.4 + - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.12 environment: NODE_ENV: "testing" steps: @@ -95,7 +95,10 @@ jobs: - ./node_modules - run: name: get-data - command: npm run index + command: | + set -x + sleep 45 + npm run index - run: name: run-test command: npm run integration-test diff --git a/config/index.js b/config/index.js index f8c5dd55..e91620be 100644 --- a/config/index.js +++ b/config/index.js @@ -18,9 +18,9 @@ function getElasticsearchUri(cloudFoundryEnv) { return elasticSearchCredentials.uri ? elasticSearchCredentials.uri - : 'http://localhost:9200'; + : 'http://elastic:changeme@localhost:9200'; } - return process.env.ES_URI ? process.env.ES_URI : 'http://localhost:9200'; + return process.env.ES_URI ? process.env.ES_URI : 'http://elastic:changeme@localhost:9200'; } /** diff --git a/indexes/repo/mapping.json b/indexes/repo/mapping.json new file mode 100644 index 00000000..3388a8f6 --- /dev/null +++ b/indexes/repo/mapping.json @@ -0,0 +1,340 @@ +{ + "repo": { + "properties": { + "repoID": { + "type": "keyword", + "normalizer": "lowercase" + }, + "agency": { + "type": "object", + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "acronym": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "website": { + "type": "keyword", + "normalizer": "lowercase" + }, + "codeUrl": { + "type": "keyword", + "normalizer": "lowercase" + }, + "requirements": { + "type": "nested", + "properties": { + "agencyWidePolicy": { + "type": "float" + }, + "openSourceRequirement": { + "type": "float" + }, + "inventoryRequirement": { + "type": "float" + }, + "schemaFormat": { + "type": "float" + }, + "overallCompliance": { + "type": "float" + } + } + } + } + }, + "measurementType": { + "type": "object", + "properties": { + "method": { + "type": "keyword" + }, + "ifOther": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + } + } + }, + "status": { + "type": "keyword", + "normalizer": "lowercase" + }, + "vcs": { + "type": "keyword", + "normalizer": "lowercase" + }, + "repositoryURL": { + "type": "keyword" + }, + "targetOperatingSystems": { + "type": "keyword", + "normalizer": "lowercase" + }, + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "version": { + "type": "keyword", + "normalizer": "lowercase" + }, + "organization": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "homepageURL": { + "type": "keyword", + "normalizer": "lowercase" + }, + "downloadURL": { + "type": "keyword", + "normalizer": "lowercase" + }, + "description": { + "type": "text", + "analyzer": "englishfulltext" + }, + "events": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "tags": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "languages": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "_fulltext": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "contact": { + "type": "object", + "include_in_root": true, + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "email": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "twitter": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "phone": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + } + } + }, + "partners": { + "type": "nested", + "include_in_root": true, + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "email": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "permissions": { + "type": "object", + "properties": { + "licenses": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "URL": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "usageType": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "exemptionText": { + "type": "text", + "analyzer": "englishfulltext" + } + } + }, + "laborHours": { + "type": "integer" + }, + "relatedCode": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "URL": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "reusedCode": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "URL": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "disclaimerURL": { + "type": "keyword", + "normalizer": "lowercase" + }, + "disclaimerText": { + "type": "text", + "analyzer": "englishfulltext" + }, + "additionalInformation": { + "type": "object", + "dynamic": true + }, + "date": { + "type": "nested", + "include_in_root": true, + "properties": { + "created": { + "type": "date", + "ignore_malformed": true + }, + "lastModified": { + "type": "date", + "ignore_malformed": true + }, + "metadataLastUpdated": { + "type": "date", + "ignore_malformed": true + } + } + } + } + } +} \ No newline at end of file diff --git a/indexes/repo/mapping_100.json b/indexes/repo/mapping_100.json deleted file mode 100644 index e8a7c01e..00000000 --- a/indexes/repo/mapping_100.json +++ /dev/null @@ -1,223 +0,0 @@ -{ - "repo": { - "properties": { - "repoID": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "acronym": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "website": { - "type": "string", - "index": "not_analyzed" - }, - "codeUrl": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "status": { - "type": "string", - "analyzer": "keyword_ci" - }, - "vcs": { - "type": "string", - "analyzer": "keyword_ci" - }, - "repository": { - "type": "string", - "analyzer": "keyword_ci" - }, - "name": { - "type": "string", - "analyzer": "keyword_ci" - }, - "homepage": { - "type": "string", - "analyzer": "keyword_ci" - }, - "downloadURL": { - "type": "string", - "index": "not_analyzed" - }, - "description": { - "type": "string", - "analyzer": "englishfulltext", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "events": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "tags": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "languages": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "updated": { - "type": "nested", - "include_in_root": true, - "properties": { - "metadataLastUpdated": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "lastCommit": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "sourceCodeLastModified": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - } - }, - "contact": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "twitter": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "phone": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "partners": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "license": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "openSourceProject": { - "type": "byte" - }, - "governmentWideReuseProject": { - "type": "byte" - }, - "closedProject": { - "type": "byte" - }, - "exemption": { - "type": "byte" - }, - "exemptionText": { - "type": "string" - } - } - } -} diff --git a/indexes/repo/mapping_101.json b/indexes/repo/mapping_101.json deleted file mode 100644 index 1c8f26ca..00000000 --- a/indexes/repo/mapping_101.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "repo": { - "properties": { - "repoID": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "status": { - "type": "string", - "analyzer": "keyword_ci" - }, - "vcs": { - "type": "string", - "analyzer": "keyword_ci" - }, - "repository": { - "type": "string", - "analyzer": "keyword_ci" - }, - "name": { - "type": "string", - "analyzer": "keyword_ci" - }, - "homepage": { - "type": "string", - "analyzer": "keyword_ci" - }, - "downloadURL": { - "type": "string", - "index": "not_analyzed" - }, - "description": { - "type": "string", - "analyzer": "englishfulltext", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "events": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "tags": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "languages": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "updated": { - "type": "nested", - "include_in_root": true, - "properties": { - "metadataLastUpdated": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "lastCommit": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - }, - "sourceCodeLastModified": { - "type": "date", - "format": "strict_date_optional_time||epoch_millis" - } - } - }, - "contact": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "twitter": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "phone": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "partners": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "license": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "openSourceProject": { - "type": "byte" - }, - "governmentWideReuseProject": { - "type": "byte" - }, - "closedProject": { - "type": "byte" - }, - "exemption": { - "type": "byte" - }, - "exemptionText": { - "type": "string" - } - } - } - } - \ No newline at end of file diff --git a/indexes/repo/mapping_200.json b/indexes/repo/mapping_200.json deleted file mode 100644 index f7f67978..00000000 --- a/indexes/repo/mapping_200.json +++ /dev/null @@ -1,492 +0,0 @@ -{ - "repo": { - "properties": { - "repoID": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "object", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "acronym": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "keyword_ci" - } - } - }, - "website": { - "type": "string", - "index": "not_analyzed" - }, - "codeUrl": { - "type": "string", - "index": "not_analyzed" - }, - "requirements": { - "type": "nested", - "properties": { - "agencyWidePolicy": { - "type": "float", - "index": "not_analyzed" - }, - "openSourceRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "inventoryRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "schemaFormat": { - "type": "float", - "index": "not_analyzed" - }, - "overallCompliance": { - "type": "float", - "index": "not_analyzed" - } - } - } - } - }, - "measurementType": { - "type": "object", - "properties": { - "method": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "ifOther": { - "type": "string", - "analyzer": "englishfulltext", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "status": { - "type": "string", - "analyzer": "keyword_ci" - }, - "vcs": { - "type": "string", - "analyzer": "keyword_ci" - }, - "repositoryURL": { - "type": "string", - "analyzer": "keyword_ci" - }, - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "version": { - "type": "string", - "analyzer": "keyword_ci" - }, - "organization": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "homepageURL": { - "type": "string", - "analyzer": "keyword_ci" - }, - "downloadURL": { - "type": "string", - "index": "not_analyzed" - }, - "description": { - "type": "string", - "analyzer": "englishfulltext" - }, - "events": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "tags": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "languages": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "contact": { - "type": "object", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "twitter": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "phone": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "partners": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "permissions": { - "type": "object", - "properties": { - "licenses": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "usageType": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "exemptionText": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "laborHours": { - "type": "integer", - "index": "not_analyzed" - }, - "relatedCode": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "reusedCode": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "disclaimerURL": { - "type": "string", - "index": "not_analyzed" - }, - "disclaimerText": { - "type": "string", - "analyzer": "englishfulltext" - }, - "date": { - "type": "nested", - "include_in_root": true, - "properties": { - "created": { - "type": "date", - "ignore_malformed": true - }, - "lastModified": { - "type": "date", - "ignore_malformed": true - }, - "metadataLastUpdated": { - "type": "date", - "ignore_malformed": true - } - } - } - } - }, - "status": { - "properties": { - "last_data_harvest": { - "type": "date" - }, - "version": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "object", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "acronym": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "keyword_ci" - } - } - }, - "website": { - "type": "string", - "index": "not_analyzed" - }, - "codeUrl": { - "type": "string", - "index": "not_analyzed" - }, - "requirements": { - "type": "object", - "properties": { - "agencyWidePolicy": { - "type": "float", - "index": "not_analyzed" - }, - "openSourceRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "inventoryRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "schemaFormat": { - "type": "float", - "index": "not_analyzed" - }, - "overallCompliance": { - "type": "float", - "index": "not_analyzed" - } - } - } - } - }, - "issues": { - "type": "nested", - "properties": { - "organization": { - "type": "string" - }, - "project_name": { - "type": "string" - }, - "errors": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - }, - "warning": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - }, - "enhancements": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - } - } - }, - "fallback_used": { - "type": "boolean" - } - } - } -} \ No newline at end of file diff --git a/indexes/repo/settings.json b/indexes/repo/settings.json index 25611df0..7787d2c3 100644 --- a/indexes/repo/settings.json +++ b/indexes/repo/settings.json @@ -1,5 +1,12 @@ { "analysis": { + "normalizer": { + "lowercase": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, "filter": { "english_stop": { "type": "stop", @@ -25,10 +32,6 @@ } }, "analyzer": { - "keyword_ci": { - "filter": "lowercase", - "tokenizer": "keyword" - }, "englishhtml": { "tokenizer": "standard", "char_filter": ["html_strip"], diff --git a/indexes/status/mapping.json b/indexes/status/mapping.json index 03ddb12d..3a264a09 100644 --- a/indexes/status/mapping.json +++ b/indexes/status/mapping.json @@ -5,62 +5,55 @@ "type": "date" }, "version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "agency": { "type": "object", "properties": { "name": { - "type": "string", - "analyzer": "keyword_ci", + "type": "text", + "analyzer": "englishfulltext", "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" + "keyword": { + "type": "keyword", + "normalizer": "lowercase" } } }, "acronym": { - "type": "string", - "analyzer": "keyword_ci", + "type": "keyword", "fields": { - "_fulltext": { - "type": "string", - "analyzer": "keyword_ci" + "keyword": { + "type": "keyword", + "normalizer": "lowercase" } } }, "website": { - "type": "string", - "index": "not_analyzed" + "type": "keyword", + "normalizer": "lowercase" }, "codeUrl": { - "type": "string", - "index": "not_analyzed" + "type": "keyword", + "normalizer": "lowercase" }, "requirements": { "type": "object", "properties": { "agencyWidePolicy": { - "type": "float", - "index": "not_analyzed" + "type": "float" }, "openSourceRequirement": { - "type": "float", - "index": "not_analyzed" + "type": "float" }, "inventoryRequirement": { - "type": "float", - "index": "not_analyzed" + "type": "float" }, "schemaFormat": { - "type": "float", - "index": "not_analyzed" + "type": "float" }, "overallCompliance": { - "type": "float", - "index": "not_analyzed" + "type": "float" } } } @@ -70,28 +63,43 @@ "type": "nested", "properties": { "organization": { - "type": "string" + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } }, "project_name": { - "type": "string" + "type": "text", + "analyzer": "englishfulltext", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } }, "errors": { "type": "nested", "properties": { "keyword": { - "type": "string" + "type": "keyword" }, "dataPath": { - "type": "string" + "type": "keyword" }, "schemaPath": { - "type": "string" + "type": "keyword" }, "params": { "type": "object" }, "message": { - "type": "string" + "type": "text", + "analyzer": "englishfulltext" } } }, @@ -99,19 +107,20 @@ "type": "nested", "properties": { "keyword": { - "type": "string" + "type": "keyword" }, "dataPath": { - "type": "string" + "type": "keyword" }, "schemaPath": { - "type": "string" + "type": "keyword" }, "params": { "type": "object" }, "message": { - "type": "string" + "type": "text", + "analyzer": "englishfulltext" } } }, @@ -119,19 +128,20 @@ "type": "nested", "properties": { "keyword": { - "type": "string" + "type": "keyword" }, "dataPath": { - "type": "string" + "type": "keyword" }, "schemaPath": { - "type": "string" + "type": "keyword" }, "params": { "type": "object" }, "message": { - "type": "string" + "type": "text", + "analyzer": "englishfulltext" } } } diff --git a/indexes/status/settings.json b/indexes/status/settings.json index 37af98ba..6612d394 100644 --- a/indexes/status/settings.json +++ b/indexes/status/settings.json @@ -1,6 +1,13 @@ { "index.mapping.total_fields.limit": 2000, "analysis": { + "normalizer": { + "lowercase": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, "filter": { "english_stop": { "type": "stop", diff --git a/indexes/term/mapping.json b/indexes/term/mapping.json index 896c8c5d..d91ec37d 100644 --- a/indexes/term/mapping.json +++ b/indexes/term/mapping.json @@ -2,21 +2,21 @@ "term": { "properties": { "term_key": { - "type": "string", - "index": "not_analyzed" + "type": "keyword", + "normalizer": "lowercase" }, "term": { - "type": "string", - "analyzer": "keyword_ci", + "type": "keyword", + "normalizer": "lowercase", "copy_to": "term_suggest" }, "term_suggest": { - "type": "string", + "type": "text", "analyzer": "autocomplete_index", "search_analyzer": "autocomplete_search" }, "term_type": { - "type": "string" + "type": "keyword" }, "count": { "type": "long" diff --git a/indexes/term/settings.json b/indexes/term/settings.json index ede28278..dd9355ae 100644 --- a/indexes/term/settings.json +++ b/indexes/term/settings.json @@ -1,5 +1,12 @@ { "analysis": { + "normalizer": { + "lowercase": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, "filter": { "autocomplete_filter": { "type": "edge_ngram", @@ -30,10 +37,6 @@ "lowercase", "asciifolding" ] - }, - "keyword_ci": { - "tokenizer": "keyword", - "filter": "lowercase" } } } diff --git a/routes/utils.js b/routes/utils.js index 5319ccee..f1d4fb8a 100644 --- a/routes/utils.js +++ b/routes/utils.js @@ -4,7 +4,7 @@ const git = require("git-rev"); const pkg = require("../package.json"); const Jsonfile = require("jsonfile"); const Utils = require('../utils'); -const repoMapping = require('../indexes/repo/mapping_201.json'); +const repoMapping = require('../indexes/repo/mapping.json'); const searchPropsByType = Utils.getFlattenedMappingPropertiesByType(repoMapping["repo"]); @@ -119,7 +119,9 @@ function getInvalidRepoQueryParams (queryParams) { let without = _.without(queryParams, "from", "size", "sort", "q", "include", "exclude"); return without.filter((queryParam) => { - if (_.includes(searchPropsByType["string"], queryParam)) { + if (_.includes(searchPropsByType["keyword"], queryParam)) { + return false; + } else if (_.includes(searchPropsByType["text"], queryParam)) { return false; } else if (queryParam.endsWith("_gte") || queryParam.endsWith("_lte")) { let paramWithoutOp = queryParam.substring(0, queryParam.length - 4); diff --git a/services/indexer/repo/index.js b/services/indexer/repo/index.js index f4d70c4e..8ad8dc2b 100644 --- a/services/indexer/repo/index.js +++ b/services/indexer/repo/index.js @@ -6,7 +6,7 @@ const AbstractIndexer = require("../abstract_indexer"); const AgencyJsonStream = require("../repo/AgencyJsonStream"); const RepoIndexerStream = require("../repo/RepoIndexStream"); -const ES_MAPPING = require("../../../indexes/repo/mapping_201.json"); +const ES_MAPPING = require("../../../indexes/repo/mapping.json"); const ES_SETTINGS = require("../../../indexes/repo/settings.json"); const ES_PARAMS = { diff --git a/services/indexer/term/index.js b/services/indexer/term/index.js index 0f084c76..dd35c4e8 100644 --- a/services/indexer/term/index.js +++ b/services/indexer/term/index.js @@ -6,7 +6,7 @@ const getConfig = require('../../../config'); // NOTE: dependent on elasticsearch repos being indexed -const ES_REPO_MAPPING = require("../../../indexes/repo/mapping_201.json"); +const ES_REPO_MAPPING = require("../../../indexes/repo/mapping.json"); const ES_REPO_SETTINGS = require("../../../indexes/repo/settings.json"); const ES_REPO_PARAMS = { "esAlias": "repos", diff --git a/services/searcher/index.js b/services/searcher/index.js index 4ebea044..0b16965a 100644 --- a/services/searcher/index.js +++ b/services/searcher/index.js @@ -3,7 +3,7 @@ const Bodybuilder = require("bodybuilder"); const moment = require("moment"); const Utils = require("../../utils"); const Logger = require("../../utils/logger"); -const repoMapping = require("../../indexes/repo/mapping_201.json"); +const repoMapping = require("../../indexes/repo/mapping.json"); const DATE_FORMAT = "YYYY-MM-DD"; const REPO_RESULT_SIZE_MAX = 10000; @@ -102,17 +102,17 @@ class Searcher { _addFullTextQuery(body, searchQuery) { const searchFields = [ - "name^10", - "name._fulltext^5", + "name^5", + "name.keyword^10", "description^2", "agency.acronym", - "agency.name^5", - "agency.name._fulltext", + "agency.name", + "agency.name.keyword^5", "permissions.usageType", "tags^3", - "tags._fulltext", - "languages^3", - "languages._fulltext" + "tags.keyword^3", + "languages", + "languages.keyword^3" ]; body.query("multi_match", 'fields', searchFields, {"query": searchQuery}, {"type": "best_fields"}); @@ -122,16 +122,16 @@ class Searcher { if (filter instanceof Array) { filter.forEach((filterElement) => { logger.info(filterElement); - body.orFilter("term", field, filterElement.toLowerCase()); + body.orFilter("term", `${field}.keyword`, filterElement.toLowerCase()); }); } else { - body.filter("term", field, filter.toLowerCase()); + body.filter("term", `${field}.keyword`, filter.toLowerCase()); } } _addStringFilters(body, queryParams) { - searchPropsByType['string'].forEach((field) => { + searchPropsByType['keyword'].forEach((field) => { if(queryParams[field]) { this._addStringFilter(body, field, queryParams[field]); } @@ -250,9 +250,9 @@ class Searcher { sortOptions.mode = item; } }); - body.sort(sortField, sortOptions); + body.sort(`${sortField}.keyword`, sortOptions); } else { - body.sort(sortField, 'asc'); + body.sort(`${sortField}.keyword`, 'asc'); } }); } diff --git a/test/unit/config-loading.test.js b/test/unit/config-loading.test.js index 06211ef5..aff3eb9c 100644 --- a/test/unit/config-loading.test.js +++ b/test/unit/config-loading.test.js @@ -12,7 +12,7 @@ describe('Load config', function() { '2.0.0' ]; - config.ES_HOST.should.be.equal('http://localhost:9200'); + config.ES_HOST.should.be.equal('http://elastic:changeme@localhost:9200'); config.USE_HSTS.should.be.equal(false); config.HSTS_MAX_AGE.should.be.equal(31536000); config.HSTS_PRELOAD.should.be.equal(false); diff --git a/test/unit/services/indexer/repo/repoIndexStream.test.js b/test/unit/services/indexer/repo/repoIndexStream.test.js index c896135c..a0c141b9 100644 --- a/test/unit/services/indexer/repo/repoIndexStream.test.js +++ b/test/unit/services/indexer/repo/repoIndexStream.test.js @@ -16,7 +16,7 @@ describe('Index given repo', function(done) { let indexer; let mockAdapter = new MockAdapter(); - const ES_MAPPING = require("../../../../../indexes/repo/mapping_201.json"); + const ES_MAPPING = require("../../../../../indexes/repo/mapping.json"); const ES_SETTINGS = require("../../../../../indexes/repo/settings.json"); const ES_PARAMS = { "esAlias": "repos", diff --git a/indexes/repo/mapping_201.json b/test/unit/test_data/mappings/mapping.json similarity index 100% rename from indexes/repo/mapping_201.json rename to test/unit/test_data/mappings/mapping.json diff --git a/test/unit/test_data/mappings/mapping_201.json b/test/unit/test_data/mappings/mapping_201.json deleted file mode 100644 index 765d29fc..00000000 --- a/test/unit/test_data/mappings/mapping_201.json +++ /dev/null @@ -1,500 +0,0 @@ -{ - "repo": { - "properties": { - "repoID": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "object", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "acronym": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "keyword_ci" - } - } - }, - "website": { - "type": "string", - "index": "not_analyzed" - }, - "codeUrl": { - "type": "string", - "index": "not_analyzed" - }, - "requirements": { - "type": "nested", - "properties": { - "agencyWidePolicy": { - "type": "float", - "index": "not_analyzed" - }, - "openSourceRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "inventoryRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "schemaFormat": { - "type": "float", - "index": "not_analyzed" - }, - "overallCompliance": { - "type": "float", - "index": "not_analyzed" - } - } - } - } - }, - "measurementType": { - "type": "object", - "properties": { - "method": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "ifOther": { - "type": "string", - "analyzer": "englishfulltext", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "status": { - "type": "string", - "analyzer": "keyword_ci" - }, - "vcs": { - "type": "string", - "analyzer": "keyword_ci" - }, - "repositoryURL": { - "type": "string", - "analyzer": "keyword_ci" - }, - "targetOperatingSystems": { - "type": "string", - "analyzer": "keyword_ci" - }, - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "version": { - "type": "string", - "analyzer": "keyword_ci" - }, - "organization": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "homepageURL": { - "type": "string", - "analyzer": "keyword_ci" - }, - "downloadURL": { - "type": "string", - "index": "not_analyzed" - }, - "description": { - "type": "string", - "analyzer": "englishfulltext" - }, - "events": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "tags": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "languages": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "contact": { - "type": "object", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "twitter": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "phone": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "partners": { - "type": "nested", - "include_in_root": true, - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "email": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - } - } - }, - "permissions": { - "type": "object", - "properties": { - "licenses": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "usageType": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "exemptionText": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "laborHours": { - "type": "integer", - "index": "not_analyzed" - }, - "relatedCode": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "reusedCode": { - "type": "nested", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "URL": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "disclaimerURL": { - "type": "string", - "index": "not_analyzed" - }, - "disclaimerText": { - "type": "string", - "analyzer": "englishfulltext" - }, - "additionalInformation": { - "type": "object", - "dynamic": true - }, - "date": { - "type": "nested", - "include_in_root": true, - "properties": { - "created": { - "type": "date", - "ignore_malformed": true - }, - "lastModified": { - "type": "date", - "ignore_malformed": true - }, - "metadataLastUpdated": { - "type": "date", - "ignore_malformed": true - } - } - } - } - }, - "status": { - "properties": { - "last_data_harvest": { - "type": "date" - }, - "version": { - "type": "string", - "index": "not_analyzed" - }, - "agency": { - "type": "object", - "properties": { - "name": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "englishfulltext" - } - } - }, - "acronym": { - "type": "string", - "analyzer": "keyword_ci", - "fields": { - "_fulltext": { - "type": "string", - "analyzer": "keyword_ci" - } - } - }, - "website": { - "type": "string", - "index": "not_analyzed" - }, - "codeUrl": { - "type": "string", - "index": "not_analyzed" - }, - "requirements": { - "type": "object", - "properties": { - "agencyWidePolicy": { - "type": "float", - "index": "not_analyzed" - }, - "openSourceRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "inventoryRequirement": { - "type": "float", - "index": "not_analyzed" - }, - "schemaFormat": { - "type": "float", - "index": "not_analyzed" - }, - "overallCompliance": { - "type": "float", - "index": "not_analyzed" - } - } - } - } - }, - "issues": { - "type": "nested", - "properties": { - "organization": { - "type": "string" - }, - "project_name": { - "type": "string" - }, - "errors": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - }, - "warning": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - }, - "enhancements": { - "type": "nested", - "properties": { - "keyword": { - "type": "string" - }, - "dataPath": { - "type": "string" - }, - "schemaPath": { - "type": "string" - }, - "params": { - "type": "object" - }, - "message": { - "type": "string" - } - } - } - } - }, - "fallback_used": { - "type": "boolean" - } - } - } -} \ No newline at end of file diff --git a/test/unit/utils.test.js b/test/unit/utils.test.js index 7e53dd94..86757939 100644 --- a/test/unit/utils.test.js +++ b/test/unit/utils.test.js @@ -14,7 +14,7 @@ describe('Testing Utils module', function () { describe('flatten mapping properties', function () { let mappings; before(function () { - mappings = require('./test_data/mappings/mapping_201.json'); + mappings = require('./test_data/mappings/mapping.json'); }) it('should return a flattened versions of the passed mapping object using getFlattenedMappingProperties', function () { const expected = {