Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | import * as fs from "fs-extra"
import * as path from "path"
import * as glob from "glob"
import { without } from "lodash"
import * as lodash from "lodash"
import * as cheerio from "cheerio"
import ProgressBar = require("progress")
import * as wpdb from "../db/wpdb"
import * as db from "../db/db"
import {
BLOG_POSTS_PER_PAGE,
BASE_DIR,
WORDPRESS_DIR,
} from "../settings/serverSettings"
import {
renderFrontPage,
renderBlogByPageNum,
renderChartsPage,
renderMenuJson,
renderSearchPage,
renderDonatePage,
entriesByYearPage,
makeAtomFeed,
feedbackPage,
renderNotFoundPage,
renderCountryProfile,
flushCache as siteBakingFlushCache,
renderPost,
} from "../baker/siteRenderers"
import {
bakeGrapherUrls,
getGrapherExportsByUrl,
GrapherExports,
} from "../baker/GrapherBakingUtils"
import { makeSitemap } from "../baker/sitemap"
import { Post } from "../db/model/Post"
import { bakeCountries } from "../baker/countryProfiles"
import { countries } from "../clientUtils/countries"
import { execWrapper } from "../db/execWrapper"
import { logErrorAndMaybeSendToSlack } from "../serverUtils/slackLog"
import { countryProfileSpecs } from "../site/countryProfileProjects"
import { getRedirects } from "./redirects"
import { bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers } from "./GrapherBaker"
import { EXPLORERS_ROUTE_FOLDER } from "../explorer/ExplorerConstants"
import { bakeEmbedSnippet } from "../site/webpackUtils"
import { FullPost } from "../clientUtils/owidTypes"
import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants"
import {
bakeAllExplorerRedirects,
bakeAllPublishedExplorers,
} from "./ExplorerBaker"
import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer"
export class SiteBaker {
private grapherExports!: GrapherExports
private bakedSiteDir: string
baseUrl: string
progressBar: ProgressBar
constructor(bakedSiteDir: string, baseUrl: string) {
this.bakedSiteDir = bakedSiteDir
this.baseUrl = baseUrl
this.progressBar = new ProgressBar(
"BakeAll [:bar] :current/:total :elapseds :name\n",
{
total: 15,
}
)
}
private async bakeEmbeds() {
// Find all grapher urls used as embeds in all posts on the site
const rows = await wpdb.singleton.query(
`SELECT post_content FROM wp_posts WHERE (post_type='page' OR post_type='post' OR post_type='wp_block') AND post_status='publish'`
)
let grapherUrls = []
for (const row of rows) {
const $ = cheerio.load(row.post_content)
grapherUrls.push(
...$("iframe")
.toArray()
.filter((el) =>
(el.attribs["src"] || "").match(/\/grapher\//)
)
.map((el) => el.attribs["src"].trim())
)
}
grapherUrls = lodash.uniq(grapherUrls)
await bakeGrapherUrls(grapherUrls)
this.grapherExports = await getGrapherExportsByUrl()
this.progressBar.tick({ name: "✅ baked embeds" })
}
private async bakeCountryProfiles() {
countryProfileSpecs.forEach(async (spec) => {
// Delete all country profiles before regenerating them
await fs.remove(`${this.bakedSiteDir}/${spec.rootPath}`)
// Not necessary, as this is done by stageWrite already
// await this.ensureDir(profile.rootPath)
for (const country of countries) {
const html = await renderCountryProfile(
spec,
country,
this.grapherExports
).catch(() =>
console.error(
`${country.name} country profile not baked for project "${spec.project}". Check that both pages "${spec.landingPageSlug}" and "${spec.genericProfileSlug}" exist and are published.`
)
)
if (html) {
const outPath = path.join(
this.bakedSiteDir,
`${spec.rootPath}/${country.slug}.html`
)
await this.stageWrite(outPath, html)
}
}
})
this.progressBar.tick({ name: "✅ baked country profiles" })
}
// Bake an individual post/page
private async bakePost(post: FullPost) {
const html = await renderPost(post, this.baseUrl, this.grapherExports)
const outPath = path.join(this.bakedSiteDir, `${post.slug}.html`)
await fs.mkdirp(path.dirname(outPath))
await this.stageWrite(outPath, html)
}
// Returns the slugs of posts which exist on the filesystem but are not in the DB anymore.
// This happens when posts have been saved in previous bakes but have been since then deleted, unpublished or renamed.
// Among all existing slugs on the filesystem, some are not coming from WP. They are baked independently and should not
// be deleted if WP does not list them (e.g. grapher/*).
private getPostSlugsToRemove(postSlugsFromDb: string[]) {
const existingSlugs = glob
.sync(`${this.bakedSiteDir}/**/*.html`)
.map((path) =>
path.replace(`${this.bakedSiteDir}/`, "").replace(".html", "")
)
.filter(
(path) =>
!path.startsWith("uploads") &&
!path.startsWith("grapher") &&
!path.startsWith("countries") &&
!path.startsWith("country") &&
!path.startsWith("blog") &&
!path.startsWith("entries-by-year") &&
!path.startsWith("explore") &&
!countryProfileSpecs.some((spec) =>
path.startsWith(spec.rootPath)
) &&
path !== "donate" &&
path !== "feedback" &&
path !== "charts" &&
path !== "search" &&
path !== "index" &&
path !== "identifyadmin" &&
path !== "404" &&
path !== "google8272294305985984"
)
return without(existingSlugs, ...postSlugsFromDb)
}
// Bake all Wordpress posts, both blog posts and entry pages
private async bakePosts() {
const postsApi = await wpdb.getPosts()
const postSlugs = []
for (const postApi of postsApi) {
const post = await wpdb.getFullPost(postApi)
postSlugs.push(post.slug)
await this.bakePost(post)
}
// Maxes out resources (TODO: RCA)
// await Promise.all(bakingPosts.map(post => this.bakePost(post)))
// Delete any previously rendered posts that aren't in the database
for (const slug of this.getPostSlugsToRemove(postSlugs)) {
const outPath = `${this.bakedSiteDir}/${slug}.html`
await fs.unlink(outPath)
this.stage(outPath, `DELETING ${outPath}`)
}
this.progressBar.tick({ name: "✅ baked posts" })
}
// Bake unique individual pages
private async bakeSpecialPages() {
await this.stageWrite(
`${this.bakedSiteDir}/index.html`,
await renderFrontPage()
)
await this.stageWrite(
`${this.bakedSiteDir}/donate.html`,
await renderDonatePage()
)
await this.stageWrite(
`${this.bakedSiteDir}/feedback.html`,
await feedbackPage()
)
await this.stageWrite(
`${this.bakedSiteDir}/charts.html`,
await renderChartsPage()
)
await this.stageWrite(
`${this.bakedSiteDir}/search.html`,
await renderSearchPage()
)
await this.stageWrite(
`${this.bakedSiteDir}/404.html`,
await renderNotFoundPage()
)
await this.stageWrite(
`${this.bakedSiteDir}/headerMenu.json`,
await renderMenuJson()
)
await this.stageWrite(
`${this.bakedSiteDir}/sitemap.xml`,
await makeSitemap()
)
const explorerAdminServer = new ExplorerAdminServer(GIT_CMS_DIR)
await bakeAllExplorerRedirects(this.bakedSiteDir, explorerAdminServer)
await bakeAllPublishedExplorers(
`${this.bakedSiteDir}/${EXPLORERS_ROUTE_FOLDER}/`,
explorerAdminServer
)
this.progressBar.tick({ name: "✅ baked special pages" })
}
// Pages that are expected by google scholar for indexing
private async bakeGoogleScholar() {
await this.stageWrite(
`${this.bakedSiteDir}/entries-by-year/index.html`,
await entriesByYearPage()
)
const rows = (await db
.knexTable(Post.table)
.where({ status: "publish" })
.join("post_tags", { "post_tags.post_id": "posts.id" })
.join("tags", { "tags.id": "post_tags.tag_id" })
.where({ "tags.name": "Entries" })
.select(db.knexRaw("distinct year(published_at) as year"))
.orderBy("year", "DESC")) as { year: number }[]
const years = rows.map((r) => r.year)
for (const year of years) {
await this.stageWrite(
`${this.bakedSiteDir}/entries-by-year/${year}.html`,
await entriesByYearPage(year)
)
}
this.progressBar.tick({ name: "✅ baked google scholar" })
}
// Bake the blog index
private async bakeBlogIndex() {
const allPosts = await wpdb.getBlogIndex()
const numPages = Math.ceil(allPosts.length / BLOG_POSTS_PER_PAGE)
for (let i = 1; i <= numPages; i++) {
const slug = i === 1 ? "blog" : `blog/page/${i}`
const html = await renderBlogByPageNum(i)
await this.stageWrite(`${this.bakedSiteDir}/${slug}.html`, html)
}
this.progressBar.tick({ name: "✅ baked blog index" })
}
// Bake the RSS feed
private async bakeRSS() {
await this.stageWrite(
`${this.bakedSiteDir}/atom.xml`,
await makeAtomFeed()
)
this.progressBar.tick({ name: "✅ baked rss" })
}
// Bake the static assets
private async bakeAssets() {
await execWrapper(
`rsync -havL --delete ${WORDPRESS_DIR}/web/app/uploads ${this.bakedSiteDir}/`
)
await execWrapper(
`rm -rf ${this.bakedSiteDir}/assets && cp -r ${BASE_DIR}/itsJustJavascript/webpack ${this.bakedSiteDir}/assets`
)
await execWrapper(
`rsync -hav --delete ${BASE_DIR}/public/* ${this.bakedSiteDir}/`
)
await fs.writeFile(
`${this.bakedSiteDir}/grapher/embedCharts.js`,
bakeEmbedSnippet(this.baseUrl)
)
this.stage(`${this.bakedSiteDir}/grapher/embedCharts.js`)
this.progressBar.tick({ name: "✅ baked assets" })
}
async bakeRedirects() {
const redirects = await getRedirects()
this.progressBar.tick({ name: "✅ got redirects" })
await this.stageWrite(
path.join(this.bakedSiteDir, `_redirects`),
redirects.join("\n")
)
this.progressBar.tick({ name: "✅ baked redirects" })
}
async bakeWordpressPages() {
await this.bakeRedirects()
await this.bakeEmbeds()
await this.bakeBlogIndex()
await this.bakeRSS()
await this.bakeAssets()
await this.bakeGoogleScholar()
await this.bakePosts()
}
private async _bakeNonWordpressPages() {
await bakeCountries(this)
await this.bakeSpecialPages()
await this.bakeCountryProfiles()
await bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers(
this.bakedSiteDir
)
this.progressBar.tick({
name: "✅ bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers",
})
}
async bakeNonWordpressPages() {
this.progressBar = new ProgressBar(
"BakeAll [:bar] :current/:total :elapseds :name\n",
{
total: 5,
}
)
await this._bakeNonWordpressPages()
}
async bakeAll() {
// Ensure caches are correctly initialized
this.flushCache()
await this.bakeWordpressPages()
await this._bakeNonWordpressPages()
this.flushCache()
}
async ensureDir(relPath: string) {
const outPath = path.join(this.bakedSiteDir, relPath)
await fs.mkdirp(outPath)
}
async writeFile(relPath: string, content: string) {
const outPath = path.join(this.bakedSiteDir, relPath)
await fs.writeFile(outPath, content)
this.stage(outPath)
}
private async stageWrite(outPath: string, content: string) {
await fs.mkdirp(path.dirname(outPath))
await fs.writeFile(outPath, content)
this.stage(outPath)
}
private stage(outPath: string, msg?: string) {
console.log(msg || outPath)
}
private async execAndLogAnyErrorsToSlack(cmd: string) {
console.log(cmd)
try {
return await execWrapper(cmd)
} catch (error) {
// Log error to Slack, but do not throw error
return logErrorAndMaybeSendToSlack(error)
}
}
async deployToNetlifyAndPushToGitPush(
commitMsg: string,
authorEmail?: string,
authorName?: string
) {
const deployDirectlyToNetlix = fs.existsSync(
path.join(this.bakedSiteDir, ".netlify/state.json")
)
const progressBar = new ProgressBar(
"DeployToNetlify [:bar] :current/:total :elapseds :name\n",
{
total: 3 + (deployDirectlyToNetlix ? 1 : 0),
}
)
progressBar.tick({ name: "✅ ready to deploy" })
// Deploy directly to Netlify (faster than using the github hook)
if (deployDirectlyToNetlix) {
await this.execAndLogAnyErrorsToSlack(
`cd ${this.bakedSiteDir} && ${BASE_DIR}/node_modules/.bin/netlify deploy -d . --prodIfUnlocked --timeout 6000`
)
progressBar.tick({ name: "✅ deployed directly to netlify" })
}
// Ensure there is a git repo in there
await this.execAndLogAnyErrorsToSlack(
`cd ${this.bakedSiteDir} && git init`
)
progressBar.tick({ name: "✅ ensured git repo" })
// Prettify HTML source for easier debugging
// Target root level HTML files only (entries and posts) for performance
// reasons.
// TODO: check again --only-changed
// await this.execWrapper(`cd ${BAKED_SITE_DIR} && ${BASE_DIR}/node_modules/.bin/prettier --write "./*.html"`)
if (authorEmail && authorName && commitMsg)
await this.execAndLogAnyErrorsToSlack(
`cd ${this.bakedSiteDir} && git add -A . && git commit --allow-empty --author='${authorName} <${authorEmail}>' -a -m '${commitMsg}' && git push origin master`
)
else
await this.execAndLogAnyErrorsToSlack(
`cd ${this.bakedSiteDir} && git add -A . && git commit --allow-empty -a -m '${commitMsg}' && git push origin master`
)
progressBar.tick({ name: "✅ committed and pushed to github" })
}
endDbConnections() {
wpdb.singleton.end()
db.closeTypeOrmAndKnexConnections()
}
private flushCache() {
// Clear caches to allow garbage collection while waiting for next run
wpdb.flushCache()
siteBakingFlushCache()
this.progressBar.tick({ name: "✅ cache flushed" })
}
}
|