Skip to content

Improving build speed by running it parallel 🚀 #44362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sanjaiyan-dev wants to merge 5 commits into from
Closed

Improving build speed by running it parallel 🚀 #44362

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0c03930
optimize: 01
sanjaiyan-dev Dec 26, 2022
0218dd9
optimization 02
sanjaiyan-dev Dec 26, 2022
500c3a2
revert
sanjaiyan-dev Dec 26, 2022
be08ea0
optimization 02
sanjaiyan-dev Dec 26, 2022
63c7f9c
optimization 3
sanjaiyan-dev Dec 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 113 additions & 122 deletions packages/next/build/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1726,61 +1726,59 @@ export default async function build(
})
}

for (let page of pageKeys.pages) {
await includeExcludeSpan
.traceChild('include-exclude', { page })
.traceAsyncFn(async () => {
const includeGlobs = pageTraceIncludes.get(page)
const excludeGlobs = pageTraceExcludes.get(page)
page = normalizePagePath(page)
await Promise.all(pageKeys.pages.map((page) => includeExcludeSpan
.traceChild('include-exclude', { page })
.traceAsyncFn(async () => {
const includeGlobs = pageTraceIncludes.get(page)
const excludeGlobs = pageTraceExcludes.get(page)
page = normalizePagePath(page)

if (!includeGlobs?.length && !excludeGlobs?.length) {
return
}

if (!includeGlobs?.length && !excludeGlobs?.length) {
return
const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(
await promises.readFile(traceFile, 'utf8')
)
let includes: string[] = []

if (includeGlobs?.length) {
for (const includeGlob of includeGlobs) {
const results = await glob(includeGlob)
includes.push(
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
})
)
}
}
const combined = new Set([...traceContent.files, ...includes])

const traceFile = path.join(
distDir,
'server/pages',
`${page}.js.nft.json`
if (excludeGlobs?.length) {
const resolvedGlobs = excludeGlobs.map((exclude) =>
path.join(dir, exclude)
)
const pageDir = path.dirname(traceFile)
const traceContent = JSON.parse(
await promises.readFile(traceFile, 'utf8')
)
let includes: string[] = []

if (includeGlobs?.length) {
for (const includeGlob of includeGlobs) {
const results = await glob(includeGlob)
includes.push(
...results.map((file) => {
return path.relative(pageDir, path.join(dir, file))
})
)
combined.forEach((file) => {
if (isMatch(path.join(pageDir, file), resolvedGlobs)) {
combined.delete(file)
}
}
const combined = new Set([...traceContent.files, ...includes])

if (excludeGlobs?.length) {
const resolvedGlobs = excludeGlobs.map((exclude) =>
path.join(dir, exclude)
)
combined.forEach((file) => {
if (isMatch(path.join(pageDir, file), resolvedGlobs)) {
combined.delete(file)
}
})
}
})
}

await promises.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
})
}
await promises.writeFile(
traceFile,
JSON.stringify({
version: traceContent.version,
files: [...combined],
})
)
})))
})

// TODO: move this inside of webpack so it can be cached
Expand Down Expand Up @@ -2455,13 +2453,10 @@ export default async function build(
}

// Only move /404 to /404 when there is no custom 404 as in that case we don't know about the 404 page
if (!hasPages404 && useStatic404) {
await moveExportedPage('/_error', '/404', '/404', false, 'html')
}

if (useDefaultStatic500) {
await moveExportedPage('/_error', '/500', '/500', false, 'html')
}
await Promise.all([
(!hasPages404 && useStatic404) && moveExportedPage('/_error', '/404', '/404', false, 'html'),
useDefaultStatic500 && moveExportedPage('/_error', '/500', '/500', false, 'html')
])

for (const page of combinedPages) {
const isSsg = ssgPages.has(page)
Expand All @@ -2486,19 +2481,13 @@ export default async function build(
// fallback is enabled. Below, we handle the specific prerenders
// of these.
const hasHtmlOutput = !(isSsg && isDynamic && !isStaticSsgFallback)
const ampPage = `${file}.amp`

if (hasHtmlOutput) {
await moveExportedPage(page, page, file, isSsg, 'html')
}

if (hasAmp && (!isSsg || (isSsg && !isDynamic))) {
const ampPage = `${file}.amp`
await moveExportedPage(page, ampPage, ampPage, isSsg, 'html')

if (isSsg) {
await moveExportedPage(page, ampPage, ampPage, isSsg, 'json')
}
}
await Promise.all([
hasHtmlOutput && moveExportedPage(page, page, file, isSsg, 'html'),
(hasAmp && (!isSsg || (isSsg && !isDynamic))) && moveExportedPage(page, ampPage, ampPage, isSsg, 'html'),
((hasAmp && (!isSsg || (isSsg && !isDynamic))) && isSsg) && moveExportedPage(page, ampPage, ampPage, isSsg, 'json')
])

if (isSsg) {
// For a non-dynamic SSG page, we must copy its data file
Expand Down Expand Up @@ -2547,42 +2536,41 @@ export default async function build(
const extraRoutes = additionalSsgPaths.get(page) || []
for (const route of extraRoutes) {
const pageFile = normalizePagePath(route)
await moveExportedPage(
page,
route,
pageFile,
isSsg,
'html',
true
)
await moveExportedPage(
page,
route,
pageFile,
isSsg,
'json',
true
)

if (hasAmp) {
const ampPage = `${pageFile}.amp`
await moveExportedPage(
const ampPage = `${pageFile}.amp`
await Promise.all([
moveExportedPage(
page,
route,
pageFile,
isSsg,
'html',
true
),
moveExportedPage(
page,
route,
pageFile,
isSsg,
'json',
true
),
hasAmp && moveExportedPage(
page,
ampPage,
ampPage,
isSsg,
'html',
true
)
await moveExportedPage(
),
hasAmp && moveExportedPage(
page,
ampPage,
ampPage,
isSsg,
'json',
true
)
}
])

finalPrerenderRoutes[route] = {
initialRevalidateSeconds:
Expand Down Expand Up @@ -2729,24 +2717,26 @@ export default async function build(
pathname: makeRe(p.pathname ?? '**').source,
}))

await promises.writeFile(
path.join(distDir, IMAGES_MANIFEST),
JSON.stringify({
version: 1,
images,
}),
'utf8'
)
await promises.writeFile(
path.join(distDir, EXPORT_MARKER),
JSON.stringify({
version: 1,
hasExportPathMap: typeof config.exportPathMap === 'function',
exportTrailingSlash: config.trailingSlash === true,
isNextImageImported: isNextImageImported === true,
}),
'utf8'
)
await Promise.all([
promises.writeFile(
path.join(distDir, IMAGES_MANIFEST),
JSON.stringify({
version: 1,
images,
}),
'utf8'
),
promises.writeFile(
path.join(distDir, EXPORT_MARKER),
JSON.stringify({
version: 1,
hasExportPathMap: typeof config.exportPathMap === 'function',
exportTrailingSlash: config.trailingSlash === true,
isNextImageImported: isNextImageImported === true,
}),
'utf8'
)
])
await promises.unlink(path.join(distDir, EXPORT_DETAIL)).catch((err) => {
if (err.code === 'ENOENT') {
return Promise.resolve()
Expand Down Expand Up @@ -2776,19 +2766,20 @@ export default async function build(
})
await promises.copyFile(filePath, outputPath)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'

await Promise.all([
recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
),
{ overwrite: true }
),
{ overwrite: true }
)
if (appDir) {
await recursiveCopy(
appDir && recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'app'),
path.join(
distDir,
Expand All @@ -2799,7 +2790,7 @@ export default async function build(
),
{ overwrite: true }
)
}
])
}

staticPages.forEach((pg) => allStaticPages.add(pg))
Expand Down