Skip to content

Commit 29d1ffe

Browse files
PatchesMapsPatchesMaps
authored
WV-3750 ignore cache flag (#6194)
* Comprehensive performance enhancements * Improve error handling and logging in layer metadata fetching * Simplify promise handling and improve error logging in extractConfigFromWMTS.js and getVisMetadata.js * Add cache mode option to build scripts and related tasks * Fix debug mode activation to correctly set NODE_DEBUG in buildOptions.sh * Enhance debug mode activation to support multiple NODE_DEBUG values * remove extraneous files
1 parent ab084f4 commit 29d1ffe

13 files changed

+134
-40
lines changed

‎package.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,14 @@
2929
"analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
3030
"build": "run-s clean getcapabilities build:config build:dev",
3131
"build:verbose": "run-s clean getcapabilities:verbose build:config:verbose build:dev",
32+
"build:debug": "run-s clean getcapabilities:debug build:config:debug build:dev",
33+
"build:nocache": "run-s clean getcapabilities:nocache build:config:nocache build:dev",
3234
"build:ci": "run-s getcapabilities build:config build:prod",
3335
"build:options": "bash tasks/buildOptions.sh",
34-
"build:options:verbose": "bash tasks/buildOptions.sh -v",
3536
"build:config": "run-s build:options && node ./tasks/util/config.js",
36-
"build:config:verbose": "run-s build:options:verbose && node ./tasks/util/config.js",
37+
"build:config:verbose": "run-s build:options -- -v && node ./tasks/util/config.js",
38+
"build:config:debug": "run-s build:options -- -d && node ./tasks/util/config.js",
39+
"build:config:nocache": "run-s build:options -- -f && node ./tasks/util/config.js",
3740
"build:dev": "cross-env NODE_ENV=development webpack",
3841
"build:prod": "cross-env NODE_ENV=production webpack",
3942
"clean": "node ./tasks/util/clean.js",
@@ -51,6 +54,8 @@
5154
"e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop",
5255
"getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'",
5356
"getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'",
57+
"getcapabilities:debug": "bash -c 'FETCH_GC=1 npm run build:options -- -d'",
58+
"getcapabilities:nocache": "bash -c 'FETCH_GC=1 npm run build:options -- -f'",
5459
"generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py",
5560
"linkcheck": "node ./tasks/link-check",
5661
"lint": "run-s lint:js lint:scss",

‎tasks/build-options/extractConfigFromWMTS.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ const options = yargs(hideBin(process.argv))
3434
type: 'string',
3535
description: 'mode'
3636
})
37+
.option('cacheMode', {
38+
demandOption: false,
39+
alias: 'cm',
40+
type: 'string',
41+
description: 'Cache mode for fetching data'
42+
})
3743
.epilog('Extracts configuration information from a WMTS GetCapabilities file, converts the XML to JSON')
3844

3945
const { argv } = options
@@ -46,6 +52,7 @@ const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))
4652
const inputDir = argv.inputDir
4753
const outputDir = argv.outputDir
4854
const mode = argv.mode
55+
const cacheMode = argv.cacheMode
4956

5057
if (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {
5158
throw new Error(`${prog}: Error: "wv-options-wmts" not in config file`)
@@ -175,7 +182,7 @@ async function processEntry (entry) {
175182

176183
async function processLayer (gcLayer, wvLayers, entry) {
177184
const ident = gcLayer['ows:Identifier']._text
178-
if (mode === 'verbose') console.warn(`${prog}: Processing layer ${ident}...`)
185+
if (mode === 'verbose') console.trace(`Processing layer ${ident}...`)
179186
if (skip.includes(ident)) {
180187
console.log(`${ident}: skipping`)
181188
throw new SkipException(ident)
@@ -194,7 +201,7 @@ async function processLayer (gcLayer, wvLayers, entry) {
194201
const dimension = gcLayer.Dimension
195202
if (dimension['ows:Identifier']._text === 'Time') {
196203
try {
197-
wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source)
204+
wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source, cacheMode)
198205
} catch (e) {
199206
console.error(e)
200207
console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)

‎tasks/build-options/fetchPreviewSnapshots.js‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ const options = yargs(hideBin(process.argv))
2727
type: 'string',
2828
description: 'features file'
2929
})
30+
.option('cacheMode', {
31+
demandOption: false,
32+
alias: 'c',
33+
type: 'string',
34+
description: 'Cache mode for fetching data'
35+
})
3036
.epilog('Fetch preview images from WV Snapshots for any layers which they are missing.')
3137

3238
const { argv } = options
@@ -37,6 +43,7 @@ if (!argv.wvJsonFile && !argv.overridesFile && !argv.featuresFile) {
3743
const wvJsonFile = argv.wvJsonFile
3844
const overridesFile = argv.overridesFile
3945
const featuresFile = argv.featuresFile
46+
const cacheMode = argv.cacheMode
4047

4148
let overrideDatesDict = {}
4249
const badSnapshots = []
@@ -250,12 +257,24 @@ async function getSnapshots (layer) {
250257
params.LAYERS = gibsLayerId
251258
}
252259

260+
let headers = {}
261+
262+
if (cacheMode === 'no-store') {
263+
const noCacheHeaders = {
264+
'Cache-Control': 'no-cache no-store',
265+
Pragma: 'no-cache',
266+
Expires: '0'
267+
}
268+
headers = Object.assign(headers, noCacheHeaders)
269+
}
270+
253271
try {
254272
const imageReq = await axios({
255273
method: 'get',
256274
url: snapshotsUrl,
257275
params,
258-
responseType: 'stream'
276+
responseType: 'stream',
277+
headers
259278
})
260279
let statusText
261280
if (imageReq.status === 200) {

‎tasks/build-options/getCapabilities.js‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ const options = yargs(hideBin(process.argv))
3131
type: 'string',
3232
description: 'mode'
3333
})
34+
.option('cacheMode', {
35+
demandOption: false,
36+
alias: 'f',
37+
type: 'string',
38+
description: 'Cache mode for fetching data'
39+
})
3440
.epilog('Pulls GetCapabilities XML and linked metadata from configured locations')
3541

3642
const { argv } = options
@@ -39,6 +45,7 @@ if (!argv.config && !argv.getcapabilities) {
3945
}
4046
const configFile = argv.config
4147
const outputDir = argv.getcapabilities
48+
const cacheMode = argv.cacheMode
4249
const colormaps = {}
4350
const vectorstyles = {}
4451
const vectordata = {}
@@ -49,6 +56,17 @@ const vectordataDir = path.join(outputDir, 'vectordata')
4956
const configData = fs.readFileSync(configFile)
5057
const config = JSON.parse(configData)
5158

59+
let headers = {}
60+
61+
if (cacheMode === 'no-store') {
62+
const noCacheHeaders = {
63+
'Cache-Control': 'no-cache no-store',
64+
Pragma: 'no-cache',
65+
Expires: '0'
66+
}
67+
headers = Object.assign(headers, noCacheHeaders)
68+
}
69+
5270
async function main () {
5371
if (!fs.existsSync(outputDir)) {
5472
fs.mkdirSync(outputDir)
@@ -81,17 +99,17 @@ async function getCapabilities () {
8199
// convert to superagent and use promises
82100
async function fetchConfigs (inputFile, outputFile) {
83101
const writer = await fs.createWriteStream(outputFile)
84-
if (argv.mode === 'verbose') console.warn(`${prog}: Fetching ${inputFile}...`)
85-
return axios({
102+
if (argv.mode === 'verbose') console.trace(`Fetching ${inputFile}...`)
103+
const response = await axios({
86104
method: 'get',
87105
url: inputFile,
88106
responseType: 'stream',
89-
timeout: 100000
90-
}).then(async (response) => {
91-
if (argv.mode === 'verbose') console.warn(`${prog}: Writing ${outputFile}...`)
92-
await response.data.pipe(writer)
93-
return finished(writer)
107+
timeout: 100000,
108+
headers
94109
})
110+
if (argv.mode === 'verbose') console.trace(`Writing ${outputFile}...`)
111+
await response.data.pipe(writer)
112+
return finished(writer)
95113
}
96114

97115
async function handleException (error, link, dir, ext, count) {
@@ -113,7 +131,7 @@ async function processVectorData (layer) {
113131
Object.values(layer['ows:Metadata']).forEach((item) => {
114132
const schemaVersion = item._attributes['xlink:role']
115133
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
116-
if (argv.mode === 'verbose') console.warn(` Processing Metadata: ${item._attributes['xlink:href']}`)
134+
if (argv.mode === 'verbose') console.trace(` Processing Metadata: ${item._attributes['xlink:href']}`)
117135
const vectorDataLink = item._attributes['xlink:href']
118136
const vectorDataFile = path.basename(vectorDataLink)
119137
const vectorDataId = path.parse(vectorDataFile).name
@@ -125,13 +143,13 @@ async function processVectorData (layer) {
125143

126144
async function processLayer (layer) {
127145
const ident = layer['ows:Identifier']._text
128-
if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}...`)
146+
if (argv.mode === 'verbose') console.trace(`Processing layer ${ident}...`)
129147
if (layer['ows:Metadata']) {
130148
if (config.skipPalettes) {
131149
console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
132150
} else {
133151
Object.values(layer['ows:Metadata']).forEach((item) => {
134-
if (argv.mode === 'verbose') console.warn(` Processing pallette: ${item._attributes['xlink:href']}`)
152+
if (argv.mode === 'verbose') console.trace(` Processing pallette: ${item._attributes['xlink:href']}`)
135153
const schemaVersion = item._attributes['xlink:role']
136154
if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
137155
const colormapLink = item._attributes['xlink:href']

‎tasks/build-options/getVisMetadata.js‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,21 @@ const options = yargs(hideBin(process.argv))
2727
type: 'string',
2828
description: 'layer-metadata/all.json file'
2929
})
30+
.option('cacheMode', {
31+
demandOption: false,
32+
alias: 'c',
33+
type: 'string',
34+
description: 'Cache mode for fetching data'
35+
})
3036
.epilog('Creates a layer-metadata file containing all layers')
3137

3238
const { argv } = options
3339
if (!argv.features && !argv.layerOrder && !argv.layerMetadata) {
3440
throw new Error('Invalid number of arguments')
3541
}
3642

43+
const cacheMode = argv.cacheMode
44+
3745
const featuresFile = argv.features
3846
let featuresData = fs.readFileSync(featuresFile)
3947
let features = JSON.parse(featuresData)
@@ -171,14 +179,26 @@ function getDAAC (metadata) {
171179
return metadata
172180
}
173181

182+
let headers = {}
183+
184+
if (cacheMode === 'no-store') {
185+
const noCacheHeaders = {
186+
'Cache-Control': 'no-cache no-store',
187+
Pragma: 'no-cache',
188+
Expires: '0'
189+
}
190+
headers = Object.assign(headers, noCacheHeaders)
191+
}
192+
174193
async function getMetadata (layerId, baseUrl, count) {
175194
if (count) console.warn(`retry #${count} for ${layerId}`)
176195
try {
177196
const response = await axios({
178197
method: 'get',
179198
url: `${baseUrl}${layerId}.json`,
180199
responseType: 'json',
181-
timeout: 10000
200+
timeout: 10000,
201+
headers
182202
})
183203
const metadata = response.data
184204
const daac = getDAAC(metadata)

‎tasks/build-options/processTemporalLayer.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,15 @@ function toList (val) {
1111
return val instanceof Array ? val : [val]
1212
}
1313

14-
async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic') {
14+
async function processTemporalLayer (wvLayer, value, source = 'GIBS:geographic', cacheMode) {
1515
const dateFormat = 'YYYY-MM-DD'
1616
const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'
17+
const fetchOpts = cacheMode === 'no-store' ? { cache: cacheMode } : undefined
1718
try {
1819
let ranges = toList(value)
1920
const describeDomainsUrl = `https://gibs.earthdata.nasa.gov/wmts/${projDict[source]}/best/1.0.0/${wvLayer.id}/default/250m/all/all.xml`
2021
try {
21-
const describeDomainsResponse = await fetch(describeDomainsUrl)
22+
const describeDomainsResponse = await fetch(describeDomainsUrl, fetchOpts)
2223
if (describeDomainsResponse?.ok) {
2324
const describeDomainsText = await describeDomainsResponse?.text?.() || ''
2425
const parser = new xml2js.Parser()

‎tasks/build-options/validateConfigs.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ async function main () {
6767
}
6868

6969
async function validateFile (filePath) {
70-
if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)
70+
if (argv.mode === 'verbose') console.trace(`Validating ${filePath}`)
7171
const layerFile = fs.readFileSync(filePath)
7272
const layer = JSON.parse(layerFile)
7373
const valid = validate(layer)

‎tasks/build-options/validateOptions.js‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ const options = yargs(hideBin(process.argv))
2020
type: 'string',
2121
description: 'config directory'
2222
})
23+
.option('cacheMode', {
24+
demandOption: false,
25+
alias: 'cm',
26+
type: 'string',
27+
description: 'Cache mode for fetching data'
28+
})
2329
.epilog('Validates and corrects the configuration files.')
2430

2531
const { argv } = options
@@ -29,6 +35,7 @@ if (!argv.optionsFile && !argv.configDir) {
2935

3036
const optionsFile = argv.optionsFile
3137
const configDir = argv.configDir
38+
const cacheMode = argv.cacheMode
3239

3340
let errorCount = 0
3441
let warningCount = 0
@@ -110,7 +117,7 @@ async function main () {
110117
}
111118
if ('temporal' in layer) {
112119
warn(`[${layerId}] GC Layer temporal values overwritten by Options`)
113-
layer = await processTemporalLayer(layer, layer.temporal)
120+
layer = await processTemporalLayer(layer, layer.temporal, undefined, cacheMode)
114121
}
115122
if (layer.futureTime) {
116123
if ('endDate' in layer) delete layer.endDate

0 commit comments

Comments
 (0)