fix: OSD: color parsing improvements 1

R/parseOSD_functions.R

@@ -73,23 +73,7 @@
   # mineral texture classes, sorted from coarse -> fine
   textures <- c('coarse sand', 'sand', 'fine sand', 'very fine sand', 'loamy coarse sand', 'loamy sand', 'loamy fine sand', 'loamy very fine sand', 'coarse sandy loam', 'sandy loam', 'fine sandy loam', 'very fine sandy loam', 'loam', 'silt loam', 'silt', 'sandy clay loam', 'clay loam', 'silty clay loam', 'sandy clay', 'silty clay', 'clay')
 
-  ## TODO: this is too greedy as 'fine sand' will be found _within_ 'fine sandy loam'
-  # https://github.com/dylanbeaudette/parse-osd/issues/10
-
-  # combine into capturing REGEX
-  # texture.regex <- paste0('(', paste(textures, collapse='|'), ')')
-  #
-  # # get matches
-  # m <- stri_match(text, regex = texture.regex, mode='first', opts_regex=list(case_insensitive=TRUE))
-  #
-  # # fail gracefully in the case of no section data or no matches
-  # if(nrow(m) < 1)
-  #   return(NA)
-  #
-  # # keep only matches and convert to lower case
-  # m <- tolower(m[, 2])
-
-  ## 2019-05-29: generalized for all non-greedy, exact matching
+  ## 2019-05-29: generalized for all non-greedy, exact matching, longest string
   m <- .findClass(needle = textures, haystack = text)
   m <- tolower(m)
 
@@ -319,35 +303,27 @@
 
 #' @importFrom stringi stri_match_all
 .extractHzData <- function(tp) {
-
-  ## REGEX rules
-  # http://regexr.com/
-  ## TODO: combine top+bottom with top only rules
-  #       "O" = "0"
-  #       "l" = "1"
-  ## ideas: http://stackoverflow.com/questions/15474741/python-regex-optional-capture-group
-
-  # expect em dashes (\u2014) used after horizon designation as of May 2023
-  # https://github.com/ncss-tech/SoilKnowledgeBase/issues/64
-
+
   # detect horizons with both top and bottom depths
   hz.rule <- "([\\^\\'\\/a-zA-Z0-9]+(?: and [\\^\\'\\/a-zA-Z0-9]+)?)\\s*[-=\u2014]+\\s*([Ol0-9.]+)\\s*?(to|-)?\\s+?([Ol0-9.]+)\\s*?(in|inches|cm|centimeters)"
-
+  
   # detect horizons with no bottom depth
   hz.rule.no.bottom <- "([\\^\\'\\/a-zA-Z0-9]+(?: and [\\^\\'\\/a-zA-Z0-9]+)?)\\s*[-=\u2014]+?\\s*([Ol0-9.]+)\\s*(to|-)?\\s*([Ol0-9.]+)?\\s*?(in|inches|cm|centimeters)"
-
-  ## TODO: toggle dry/moist assumption:
-  ##
-  ## Colors are for dry soil unless otherwise stated | Colors are for moist soil unless otherwise stated
+
+  ## default encoding of colors: Toggle dry/moist assumption
   ##
-  ## E1--7 to 12 inches; very dark gray (10YR 3/1) silt loam, 50 percent gray (10YR 5/1) and 50 percent gray (10YR 6/1) dry; moderate thin platy structure parting to weak thin platy; friable, soft; common fine and medium roots throughout; common fine tubular pores; few fine distinct dark yellowish brown (10YR 4/6) friable masses of iron accumulations with sharp boundaries on faces of peds; strongly acid; clear wavy boundary.
-
+  ## Profile-level statement: Colors are for dry soil unless otherwise stated | Colors are for moist soil unless otherwise stated
+  ## 
+  ## Examples:
+  ## moist:
+  ##   E1--7 to 12 inches; very dark gray (10YR 3/1) silt loam, 50 percent gray (10YR 5/1) and 50 percent gray (10YR 6/1) dry; moderate thin platy structure parting to weak thin platy; friable, soft; common fine and medium roots throughout; common fine tubular pores; few fine distinct dark yellowish brown (10YR 4/6) friable masses of iron accumulations with sharp boundaries on faces of peds; strongly acid; clear wavy boundary.
+  ## 
+  ## dry:
   ##   A--0 to 6 inches; light gray (10YR 7/2) loam, dark grayish brown (10YR 4/2) moist; moderate coarse subangular blocky structure; slightly hard, friable, slightly sticky and slightly plastic; many very fine roots; many very fine and few fine tubular and many very fine interstitial pores; 10 percent pebbles; strongly acid (pH 5.1); clear wavy boundary. (1 to 8 inches thick)
   ##
+  dry.is.default <- length(grep('for[ athe]+(?:air-* *)?dr[yied]+[ \\n,]+(colors|soil|conditions)', tp, ignore.case = TRUE)) > 0
+  moist.is.default <- length(grep('for[ athe]+(wet|moi*st)[ \\n,]+(rubbed|crushed|broken|interior|soil|conditions)', tp, ignore.case = TRUE)) > 0
 
-  # establish default encoding of colors
-  dry.is.default <- length(grep('for dry (soil|conditions)', tp, ignore.case = TRUE)) > 0
-  moist.is.default <- length(grep('for moist (soil|conditions)', tp, ignore.case = TRUE)) > 0
 
   if (dry.is.default)
     default.moisture.state <- 'dry'
@@ -365,16 +341,9 @@
   ## TODO: account for l,O style OCR errors
   # https://github.com/ncss-tech/SoilKnowledgeBase/issues/53
 
-  ## TODO: test this
-  # get all colors matching our rule, moist and dry and unknown, 5th column is moisture state
-  # interpretation is tough when multiple colors / hz are given
   # single rule, with dry/moist state
   # note that dry/moist may not always be present
-  color.rule <- "\\(([Ol0-9]?[\\.]?[Ol0-9]?[B|G|Y|R|N]+)([ ]+?[Ol0-9\\.]+)/([Ol0-9]*)\\)\\s?(dry|moist|)"
-
-  # detect moist and dry colors
-  dry.color.rule <- "\\(([Ol0-9]?[\\.]?[Ol0-9]?[B|G|Y|R|N]+)([ ]+?[Ol0-9\\.]+)/([Ol0-9]*)\\)(?! moist)"
-  moist.color.rule <- "\\(([Ol0-9]?[\\.]?[Ol0-9]?[B|G|Y|R|N]+)([ ]+?[Ol0-9\\.]+)/([Ol0-9]*)\\) moist"
+  color.rule <- "\\(([Ol0-9]?[\\.]?[Ol0-9]?[B|G|Y|R|N]+) *([Ol0-9\\.]+) */([Ol0-9]*) *\\)\\s?(dry|moist|)"
 
   # ID actual lines of horizon information
   hz.idx <- unique(c(grep(hz.rule, tp), grep(hz.rule.no.bottom, tp)))
@@ -425,10 +394,11 @@
     ## TODO: test this!
     # parse ALL colors, result is a multi-row matrix, 5th column is moisture state
     colors <- stringi::stri_match_all(this.chunk, regex = color.rule)[[1]]
+
     # replace missing moisture state with (parsed) default value
     colors[, 5][which(colors[, 5] == '')] <- default.moisture.state
 
-    # exctract dry|moist colors, note that there may be >1 color per state
+    # extract dry|moist colors, note that there may be >1 color per state
     dc <- colors[which(colors[, 5] == 'dry'), 1:4, drop = FALSE]
     mc <- colors[which(colors[, 5] == 'moist'), 1:4, drop = FALSE]
 

inst/extdata/OSD-error-reporting/RO/auburn-hz.csv

@@ -5,6 +5,10 @@
 "ALDERFLATS","Btgl",13,30,NA,NA,NA,"5Y","5","2"
 "ANDRY","022",15,0,NA,NA,NA,"10YR","3","2"
 "ANDRY","021",30,15,NA,NA,NA,"10YR","3","2"
+"ASHLAR","A",5,13,NA,NA,NA,"l0YR","3","2"
+"ASHLAR","E",13,28,NA,NA,NA,"l0YR","5","2"
+"ASHLAR","Bw",28,51,NA,NA,NA,"l0YR","5","4"
+"ASHLAR","C",51,71,NA,NA,NA,"l0YR","5","3"
 "BAHIAHONDA","Cgl",89,173,NA,NA,NA,"10YR","8","1"
 "BEECH GROVE","Oi",3,0,NA,NA,NA,NA,NA,NA
 "BELHAVEN","Oal",23,33,NA,NA,NA,"5YR","2.5","2"
@@ -51,7 +55,9 @@
 "OKEECHOBEE","0a",20,71,NA,NA,NA,"5YR","2","1"
 "OPENLAKE","Bgl",18,33,NA,NA,NA,"10YR","4","2"
 "ORENDA","0i",0,3,NA,NA,NA,NA,NA,NA
-"PAILO","Btl",43,89,NA,NA,NA,NA,NA,NA
+"PAILO","A",3,8,NA,NA,NA,"l0YR","3","2"
+"PAILO","BE",8,43,NA,NA,NA,"l0YR","5","6"
+"PAILO","Btl",43,89,NA,NA,NA,"l0YR","5","6"
 "PAMUNKEY","2C",117,203,NA,NA,NA,"1OYR","5","6"
 "PARTLOW","Btgl",38,53,NA,NA,NA,"10YR","5","1"
 "PETAL","Btl",20,43,NA,NA,NA,"5YR","5","8"

inst/extdata/OSD-error-reporting/RO/auburn-series.csv

@@ -10,6 +10,7 @@
 "SOCO",189881,FALSE,2001,FALSE,TRUE
 "TALLADEGA",155342,FALSE,2001,FALSE,TRUE
 "GRITNEY",129163,FALSE,1997,FALSE,TRUE
+"ASHLAR",107187,TRUE,2017,FALSE,TRUE
 "VICKSBURG",103571,FALSE,2002,FALSE,TRUE
 "REMLIK",103503,FALSE,2002,FALSE,TRUE
 "STECOAH",95927,FALSE,2001,FALSE,TRUE

inst/extdata/OSD-error-reporting/RO/bozeman-hz.csv

@@ -89,7 +89,7 @@
 "CARGILL","C2ca",89,76,"2.5Y","7","4",NA,NA,NA
 "CARIBOURIDGE","0i",0,3,NA,NA,NA,NA,NA,NA
 "CARIBOURIDGE","0e",3,5,NA,NA,NA,NA,NA,NA
-"CASTELLO","Clca",23,84,NA,NA,NA,"7.5YR","8","2"
+"CASTELLO","Clca",23,84,"7.5YR","8","2","7.5YR","7","2"
 "CATAMOUNT","0i",0,3,NA,NA,NA,NA,NA,NA
 "CATLA","Cdl",41,94,"10YR","7","2","10YR","6","3"
 "CHICANE","A12",13,36,"lOYR","3","2","lOYR","2","1"
@@ -360,7 +360,7 @@
 "OLEO","0i",0,3,NA,NA,NA,NA,NA,NA
 "OLIAGA","Clca",20,46,"2.5Y","6","2","2.5Y","4","2"
 "ORAID","0i",0,3,NA,NA,NA,NA,NA,NA
-"PACKERJOHN","Al",5,25,NA,NA,NA,"10YR","4","3"
+"PACKERJOHN","Al",5,25,"10YR","4","3","10YR","3","3"
 "PALMICH","A",5,15,"l0YR","6",NA,"l0YR","3","2"
 "PALMICH","AB",15,71,"l0YR","6","2","l0YR","3","3"
 "PALMICH","Bw2",71,117,"l0YR","6","3","l0YR","4","4"
@@ -406,7 +406,7 @@
 "PUGET","Bg2",NA,64,"5Y","6","2","2.5Y","5","2"
 "PUGET","Bg4",NA,102,"5Y","7",NA,"2.5Y","5","2"
 "PUYALLUP","2Cl",46,69,"10YR","5","2","10YR","3","3"
-"QUAILRIDGE","Btl",10,20,NA,NA,NA,"10YR","4","3"
+"QUAILRIDGE","Btl",10,20,"10YR","4","3","10YR","3","3"
 "QUIETUS","0e",0,5,NA,NA,NA,NA,NA,NA
 "QUILLAYUTE","01",3,20,NA,NA,NA,NA,NA,NA
 "RACKER","0a",0,3,NA,NA,NA,NA,NA,NA
@@ -594,8 +594,8 @@
 "TRADEDOLLAR","Bs3",NA,56,"5YR","5","6","5YR","4","6"
 "TREEN","0i",0,5,NA,NA,NA,NA,NA,NA
 "TREON","A",0,18,"10YR","5","3","l0YR","3","3"
-"TRIPOD","Al",3,15,NA,NA,NA,"10YR","4","2"
-"TRIPOD","2Cl",58,127,NA,NA,NA,"10YR","7","2"
+"TRIPOD","Al",3,15,"10YR","4","2","10YR","3","2"
+"TRIPOD","2Cl",58,127,"10YR","7","2","2.5Y","5","2"
 "TROUTVILLE","0i",0,8,NA,NA,NA,NA,NA,NA
 "TROUTVILLE","0e",8,10,NA,NA,NA,NA,NA,NA
 "TRUEFISSURE","0i",0,3,NA,NA,NA,NA,NA,NA
@@ -617,7 +617,7 @@
 "VAILTON","C",102,114,"l0YR","6","4","l0YR","4","4"
 "VAILTON","Cr",114,NA,NA,NA,NA,NA,NA,NA
 "VAILTON","Bwl",NA,71,"l0YR","6","4","l0YR","4","4"
-"VAN DUSEN","Al",0,20,NA,NA,NA,"10YR","4","1"
+"VAN DUSEN","Al",0,20,"10YR","4","1","10YR","2","2"
 "VASSAR","0i",0,3,NA,NA,NA,NA,NA,NA
 "VASSAR","0e",3,5,"10YR","4","1","10YR","2","2"
 "VERNONIA","0",3,10,NA,NA,NA,NA,NA,NA