Change conversion mechanism to create better markdown files

Containerfile

@@ -36,7 +36,6 @@ ARG NUM_WORKERS=1
 ARG RHOSO_CA_CERT_URL=""
 ARG RHOSO_DOCS_GIT_URL=""
 ARG RHOSO_DOCS_GIT_BRANCH=""
-ARG RHOSO_DOCS_ATTRIBUTES_FILE_URL=""
 ARG RHOSO_RELNOTES_GIT_URL=""
 ARG RHOSO_RELNOTES_GIT_BRANCH=""
 ARG RHOSO_EXCLUDE_TITLES=""
@@ -46,7 +45,6 @@ ENV NUM_WORKERS=$NUM_WORKERS
 ENV RHOSO_CA_CERT_URL=$RHOSO_CA_CERT_URL
 ENV RHOSO_DOCS_GIT_URL=$RHOSO_DOCS_GIT_URL
 ENV RHOSO_DOCS_GIT_BRANCH=$RHOSO_DOCS_GIT_BRANCH
-ENV RHOSO_DOCS_ATTRIBUTES_FILE_URL=$RHOSO_DOCS_ATTRIBUTES_FILE_URL
 ENV RHOSO_RELNOTES_GIT_URL=$RHOSO_RELNOTES_GIT_URL
 ENV RHOSO_RELNOTES_GIT_BRANCH=$RHOSO_RELNOTES_GIT_BRANCH
 
@@ -61,8 +59,11 @@ COPY ./okp-content ./okp-content
 # * Graphviz is needed to generate text documentation for octavia
 # * python-devel and pcre-devel are needed for python-openstackclient
 # * python-devel was already installed in our base image
+# TODO: Make filter work with latest pandoc version (3.8.2) and update version
 RUN if [ ! -z "${RHOSO_DOCS_GIT_URL}" ]; then \
-        microdnf install -y graphviz pcre-devel && \
+        microdnf install -y graphviz pcre-devel tar pip && \
+        pip install lxml && \
+        bash -c 'curl -L https://github.com/jgm/pandoc/releases/download/3.1.11.1/pandoc-3.1.11.1-linux-amd64.tar.gz | tar -zx --strip-components=1 -C /usr/local/' && \
         ./scripts/get_rhoso_plaintext_docs.sh; \
     fi
 

Makefile

@@ -8,7 +8,6 @@ PRUNE_PATHS                    ?= ""
 INDEX_NAME                     ?= os-docs-$(OS_VERSION)
 RHOSO_DOCS_GIT_URL             ?= ""
 RHOSO_DOCS_GIT_BRANCH          ?= ""
-RHOSO_DOCS_ATTRIBUTES_FILE_URL ?= ""
 RHOSO_RELNOTES_GIT_URL         ?= ""
 RHOSO_RELNOTES_GIT_BRANCH      ?= ""
 RHOSO_CA_CERT_URL              ?= ""
@@ -43,7 +42,6 @@ build-image-os: ## Build a openstack rag-content container image
 	--build-arg PRUNE_PATHS=$(PRUNE_PATHS) \
 	--build-arg RHOSO_DOCS_GIT_URL=$(RHOSO_DOCS_GIT_URL) \
 	--build-arg RHOSO_DOCS_GIT_BRANCH=$(RHOSO_DOCS_GIT_BRANCH) \
-	--build-arg RHOSO_DOCS_ATTRIBUTES_FILE_URL=$(RHOSO_DOCS_ATTRIBUTES_FILE_URL) \
 	--build-arg RHOSO_RELNOTES_GIT_URL=$(RHOSO_RELNOTES_GIT_URL) \
 	--build-arg RHOSO_RELNOTES_GIT_BRANCH=$(RHOSO_RELNOTES_GIT_BRANCH) \
 	--build-arg RHOSO_CA_CERT_URL=$(RHOSO_CA_CERT_URL) \

requirements.txt

@@ -1,2 +1,3 @@
 lightspeed-rag-content @ git+https://github.com/lightspeed-core/rag-content@main
 packaging
+lxml

scripts/filters/fix-codeblock-tables.lua

@@ -0,0 +1,83 @@
+-- fix-codeblock-tables.lua
+-- AI: File generated by Cursor
+-- Ensures code blocks with table-like content maintain proper indentation
+--
+-- When pandoc converts DocBook to markdown, it sometimes outputs lines starting
+-- with `|` without proper indentation in code blocks. This happens because `|`
+-- is the markdown pipe table delimiter, and pandoc tries not to interfere with it.
+-- This filter fixes the indentation at the AST level before markdown generation.
+
+function CodeBlock(elem)
+    local text = elem.text
+    local lines = {}
+
+    -- Split into lines
+    for line in text:gmatch("([^\n]*)\n?") do
+        if line ~= "" or #lines > 0 then
+            table.insert(lines, line)
+        end
+    end
+
+    -- Check if this looks like a table (has lines with +- borders and | rows)
+    local has_table_rows = false
+    local has_border = false
+
+    for _, line in ipairs(lines) do
+        -- Check for table rows (lines starting with | after optional whitespace)
+        if line:match("^%s*|") then
+            has_table_rows = true
+        end
+        -- Check for table borders (lines with +- pattern)
+        if line:match("^%s*%+%-") then
+            has_border = true
+        end
+    end
+
+    -- If it's a table-like block, normalize all indentation
+    if has_table_rows and has_border then
+        -- Find the indentation of border lines (they should have correct indentation)
+        local border_indent = ""
+        for _, line in ipairs(lines) do
+            local indent = line:match("^(%s*)%+%-")
+            if indent then
+                border_indent = indent
+                break
+            end
+        end
+
+        -- Apply same indentation to all table-related lines
+        local fixed_lines = {}
+        for _, line in ipairs(lines) do
+            local stripped = line:match("^%s*(.*)$")
+
+            -- Check if this is a table-related line
+            local is_table_line = stripped:match("^[|+]") or
+                                  stripped == "..." or
+                                  stripped == "…"
+
+            if is_table_line then
+                -- Check current indentation
+                local current_indent = line:match("^(%s*)")
+
+                -- Only fix if indentation is significantly less than border indent
+                if #current_indent < #border_indent - 2 then
+                    table.insert(fixed_lines, border_indent .. stripped)
+                else
+                    table.insert(fixed_lines, line)
+                end
+            else
+                -- Not a table line, keep as-is
+                table.insert(fixed_lines, line)
+            end
+        end
+
+        elem.text = table.concat(fixed_lines, "\n")
+    end
+
+    return elem
+end
+
+return {{CodeBlock = CodeBlock}}
+
+
+