Skip to content

Commit d83519d

Browse files
chrisbbreuerchrisbbreuer
committed
chore: wip
1 parent 32682c4 commit d83519d

5 files changed

Lines changed: 327 additions & 88 deletions

File tree

‎packages/dtsx/bin/cli.ts‎

Lines changed: 132 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,6 @@ if (_cmd === 'stdin' || _cmd === 'emit' || _cmd === '--project') {
4141
mkdirSync(out, { recursive: true })
4242
const files = readdirSync(dir).filter((f: string) => f.endsWith('.ts') && !f.endsWith('.d.ts'))
4343

44-
// Direct imports — skip processSourceDirect wrapper for less call overhead
45-
const { scanDeclarations } = await import('../src/extractor/scanner')
46-
const { processDeclarations } = await import('../src/processor')
47-
4844
// Pre-compute all read + output paths once
4945
const n = files.length
5046
const inPaths: string[] = new Array(n)
@@ -57,44 +53,148 @@ if (_cmd === 'stdin' || _cmd === 'emit' || _cmd === '--project') {
5753
const importOrder = ['bun']
5854
const shouldUseWorkers = isBunRuntime && n >= 64
5955
let usedWorkers = false
56+
let sourcePromises: Promise<string>[] | null = null
6057

6158
if (shouldUseWorkers) {
6259
let pool: Awaited<ReturnType<(typeof import('../src/worker'))['createWorkerPool']>> | null = null
6360
try {
64-
const { createWorkerPool } = await import('../src/worker')
61+
const { createWorkerPool, calculateOptimalBatchSize } = await import('../src/worker')
6562
const { cpus } = await import('node:os')
6663
const maxWorkers = Math.max(1, cpus().length - 1)
67-
const maxInFlight = Math.max(2, maxWorkers * 2)
64+
const batchSize = Math.min(64, Math.max(8, calculateOptimalBatchSize(n, maxWorkers) * 2))
65+
const useWorkerRead = n >= 256
66+
const useWorkerWrite = true
67+
const useArrayPayload = n >= 256
68+
const shouldPrefetchSources = !useWorkerRead && n >= 256
69+
if (shouldPrefetchSources) {
70+
sourcePromises = new Array(n)
71+
for (let i = 0; i < n; i++) {
72+
sourcePromises[i] = bun.file(inPaths[i]).text()
73+
}
74+
}
6875

69-
pool = createWorkerPool({ maxWorkers, initialWorkers: maxWorkers })
76+
const batchCount = Math.ceil(n / batchSize)
77+
const maxInFlight = Math.min(batchCount, Math.max(2, maxWorkers * 2))
78+
79+
pool = createWorkerPool({
80+
maxWorkers,
81+
initialWorkers: maxWorkers,
82+
recycleAfter: Math.max(100, batchCount + 1),
83+
})
7084
await pool.init()
7185
const poolInstance = pool
7286

73-
let nextIndex = 0
87+
let nextBatch = 0
88+
const batchRunId = Date.now()
7489
const runNext = async () => {
7590
for (;;) {
76-
const idx = nextIndex++
77-
if (idx >= n)
91+
const batchIndex = nextBatch++
92+
if (batchIndex >= batchCount)
7893
return
7994

80-
const source = await bun.file(inPaths[idx]).text()
81-
const result = await poolInstance.submit({
82-
id: `task-${idx}-${Date.now()}`,
83-
type: 'process',
84-
filePath: inPaths[idx],
85-
sourceCode: source,
86-
config: {
87-
keepComments: true,
88-
importOrder,
89-
isolatedDeclarations: isoDecl,
90-
},
91-
})
95+
const batchStart = batchIndex * batchSize
96+
if (batchStart >= n)
97+
return
98+
const batchEnd = Math.min(n, batchStart + batchSize)
99+
const batchLen = batchEnd - batchStart
100+
let result: Awaited<ReturnType<typeof poolInstance.submit>>
101+
if (useArrayPayload) {
102+
const batchFilePaths: string[] = new Array(batchLen)
103+
const batchOutPaths: string[] = new Array(batchLen)
104+
let batchSources: string[] | undefined
105+
if (useWorkerRead) {
106+
for (let i = 0; i < batchLen; i++) {
107+
const idx = batchStart + i
108+
batchFilePaths[i] = inPaths[idx]
109+
batchOutPaths[i] = outPaths[idx]
110+
}
111+
}
112+
else {
113+
const readPromises: Promise<string>[] = new Array(batchLen)
114+
for (let i = 0; i < batchLen; i++) {
115+
const idx = batchStart + i
116+
batchFilePaths[i] = inPaths[idx]
117+
batchOutPaths[i] = outPaths[idx]
118+
readPromises[i] = sourcePromises ? sourcePromises[idx] : bun.file(inPaths[idx]).text()
119+
}
120+
batchSources = await Promise.all(readPromises)
121+
}
122+
123+
result = await poolInstance.submit({
124+
id: `batch-${batchRunId}-${batchIndex}`,
125+
type: 'process-batch',
126+
filePath: batchFilePaths[0] || '',
127+
filePaths: batchFilePaths,
128+
sources: batchSources,
129+
outPaths: batchOutPaths,
130+
writeOutput: useWorkerWrite,
131+
config: {
132+
keepComments: true,
133+
importOrder,
134+
isolatedDeclarations: isoDecl,
135+
},
136+
})
137+
}
138+
else {
139+
const batchFilesToProcess: Array<{ filePath: string, sourceCode?: string, outPath?: string }> = new Array(batchLen)
140+
if (useWorkerRead) {
141+
for (let i = 0; i < batchLen; i++) {
142+
const idx = batchStart + i
143+
batchFilesToProcess[i] = { filePath: inPaths[idx], outPath: outPaths[idx] }
144+
}
145+
}
146+
else {
147+
const readPromises: Promise<string>[] = new Array(batchLen)
148+
for (let i = 0; i < batchLen; i++) {
149+
const idx = batchStart + i
150+
readPromises[i] = sourcePromises ? sourcePromises[idx] : bun.file(inPaths[idx]).text()
151+
}
152+
const sources = await Promise.all(readPromises)
153+
for (let i = 0; i < batchLen; i++) {
154+
const idx = batchStart + i
155+
batchFilesToProcess[i] = { filePath: inPaths[idx], sourceCode: sources[i], outPath: outPaths[idx] }
156+
}
157+
}
92158

93-
if (!result.success || !result.content) {
94-
throw new Error(result.error || `Worker failed for ${inPaths[idx]}`)
159+
result = await poolInstance.submit({
160+
id: `batch-${batchRunId}-${batchIndex}`,
161+
type: 'process-batch',
162+
filePath: inPaths[batchStart] || '',
163+
files: batchFilesToProcess,
164+
writeOutput: useWorkerWrite,
165+
config: {
166+
keepComments: true,
167+
importOrder,
168+
isolatedDeclarations: isoDecl,
169+
},
170+
})
95171
}
96172

97-
await bun.write(outPaths[idx], result.content)
173+
if (!result.success) {
174+
throw new Error(result.error || `Worker batch failed for ${inPaths[batchStart]}`)
175+
}
176+
177+
if (useWorkerWrite) {
178+
if (result.batchResults?.length) {
179+
const firstError = result.batchResults[0]
180+
throw new Error(firstError?.error || result.error || `Worker failed for ${inPaths[batchStart]}`)
181+
}
182+
}
183+
else {
184+
if (!result.batchResults || result.batchResults.length !== batchLen) {
185+
throw new Error(result.error || `Worker batch failed for ${inPaths[batchStart]}`)
186+
}
187+
const writePromises: Promise<unknown>[] = new Array(batchLen)
188+
for (let i = 0; i < batchLen; i++) {
189+
const output = result.batchResults[i]
190+
if (!output?.success || output.content == null) {
191+
throw new Error(output?.error || `Worker failed for ${inPaths[batchStart + i]}`)
192+
}
193+
writePromises[i] = bun.write(outPaths[batchStart + i], output.content)
194+
}
195+
196+
await Promise.all(writePromises)
197+
}
98198
}
99199
}
100200

@@ -116,14 +216,16 @@ if (_cmd === 'stdin' || _cmd === 'emit' || _cmd === '--project') {
116216
}
117217

118218
if (!usedWorkers) {
219+
// Direct imports — skip processSourceDirect wrapper for less call overhead
220+
const { scanDeclarations } = await import('../src/extractor/scanner')
221+
const { processDeclarations } = await import('../src/processor')
222+
119223
// Phase 1: Read all sources into memory
120224
const sources: string[] = new Array(n)
121225
if (isBunRuntime) {
122-
const readPromises: Promise<string>[] = new Array(n)
123-
for (let i = 0; i < n; i++) {
124-
readPromises[i] = bun.file(inPaths[i]).text()
125-
}
126-
const readResults = await Promise.all(readPromises)
226+
const readResults = sourcePromises
227+
? await Promise.all(sourcePromises)
228+
: await Promise.all(inPaths.map(path => bun.file(path).text()))
127229
for (let i = 0; i < n; i++) {
128230
sources[i] = readResults[i]
129231
}

‎packages/dtsx/src/extractor/index.ts‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -114,16 +114,9 @@ export function extractDeclarations(
114114

115115
declarationCache.set(cacheKey, { declarations, contentHash, lastAccess: now })
116116

117-
// Evict oldest entry if cache exceeds max size
117+
// Evict oldest entry if cache exceeds max size (FIFO via Map insertion order)
118118
if (declarationCache.size > MAX_DECLARATION_CACHE_SIZE) {
119-
let oldestKey: string | null = null
120-
let oldestTime = Infinity
121-
for (const [key, entry] of declarationCache) {
122-
if (entry.lastAccess < oldestTime) {
123-
oldestTime = entry.lastAccess
124-
oldestKey = key
125-
}
126-
}
119+
const oldestKey = declarationCache.keys().next().value
127120
if (oldestKey) {
128121
declarationCache.delete(oldestKey)
129122
}

‎packages/dtsx/src/generator.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -603,9 +603,14 @@ async function processFileWithStatsFromSource(
603603
declarations = await pluginManager.runOnDeclarations(filePath, processedSource, declarations)
604604
}
605605

606-
// Count imports and exports
607-
const importCount = declarations.filter(d => d.kind === 'import').length
608-
const exportCount = declarations.filter(d => d.kind === 'export' || d.isExported).length
606+
// Count imports and exports (single pass, no temporary arrays)
607+
let importCount = 0
608+
let exportCount = 0
609+
for (let i = 0; i < declarations.length; i++) {
610+
const d = declarations[i]
611+
if (d.kind === 'import') importCount++
612+
if (d.kind === 'export' || d.isExported) exportCount++
613+
}
609614

610615
// Create processing context
611616
const context: ProcessingContext = {

‎packages/dtsx/src/processor/imports.ts‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -115,22 +115,6 @@ export function extractAllImportedItems(importText: string): string[] {
115115

116116
const items: string[] = []
117117

118-
// Helper to clean import item names and extract alias if present
119-
// For 'SomeType as AliasedType', returns 'AliasedType' (the local name used in code)
120-
const cleanImportItem = (item: string): string => {
121-
let trimmed = item.trim()
122-
// Remove 'type ' prefix
123-
if (trimmed.startsWith('type ')) {
124-
trimmed = trimmed.slice(5).trim()
125-
}
126-
// Handle aliases: 'OriginalName as AliasName' -> 'AliasName'
127-
const asIndex = trimmed.indexOf(' as ')
128-
if (asIndex !== -1) {
129-
return trimmed.slice(asIndex + 4).trim()
130-
}
131-
return trimmed
132-
}
133-
134118
// Find 'from' keyword position
135119
const fromIndex = importText.indexOf(' from ')
136120
if (fromIndex === -1) {
@@ -168,10 +152,21 @@ export function extractAllImportedItems(importText: string): string[] {
168152
items.push(beforeBrace)
169153
}
170154

171-
// Extract named imports from braces
155+
// Extract named imports from braces (inline clean to avoid closure allocation)
172156
const namedPart = importPart.slice(braceStart + 1, braceEnd)
173-
const namedItems = namedPart.split(',').map(cleanImportItem).filter(Boolean)
174-
items.push(...namedItems)
157+
const rawItems = namedPart.split(',')
158+
for (let ri = 0; ri < rawItems.length; ri++) {
159+
let trimmed = rawItems[ri].trim()
160+
if (!trimmed) continue
161+
if (trimmed.startsWith('type ')) {
162+
trimmed = trimmed.slice(5).trim()
163+
}
164+
const asIndex = trimmed.indexOf(' as ')
165+
if (asIndex !== -1) {
166+
trimmed = trimmed.slice(asIndex + 4).trim()
167+
}
168+
if (trimmed) items.push(trimmed)
169+
}
175170
}
176171
else {
177172
// Default import only: import defaultName from 'module'

0 commit comments

Comments
 (0)