Recursive install of node_modules in Pulumi resources

Try implementing the solution below and see if it helps:

Monorepo Package Configuration

{
  "name": "pulumi-infrastructure",
  "private": true,
  "workspaces": [
    "infra/*",
    "lib/*"
  ],
  "scripts": {
    "install-all": "npm install && npm run install:jobs",
    "install:jobs": "node scripts/install-jobs.js",
    "pulumi:up-all": "node scripts/pulumi-up-all.js"
  },
  "devDependencies": {
    "@types/node": "^18.19.69",
    "@typescript-eslint/eslint-plugin": "^8.19.0",
    "@typescript-eslint/parser": "^8.19.0",
    "eslint": "^9.17.0",
    "typescript": "^5.7.2"
  }
}

NPM Install Script

// scripts/install-jobs.js
const { execSync } = require('child_process');
const path = require('path');
const fs = require('fs');

// Directories to process
const directories = ['infra', 'lib'];

function installDependencies(dir) {
  const items = fs.readdirSync(dir);
  
  items.forEach(item => {
    const fullPath = path.join(dir, item);
    const stats = fs.statSync(fullPath);
    
    if (stats.isDirectory()) {
      const packageJsonPath = path.join(fullPath, 'package.json');
      
      if (fs.existsSync(packageJsonPath)) {
        console.log(`Installing dependencies in ${fullPath}`);
        try {
          execSync('npm install', {
            cwd: fullPath,
            stdio: 'inherit'
          });
        } catch (error) {
          console.error(`Error installing dependencies in ${fullPath}:`, error);
          process.exit(1);
        }
      }
    }
  });
}

// Process each directory
directories.forEach(dir => {
  console.log(`Processing directory: ${dir}`);
  installDependencies(dir);
});

Pulumi Up Script

// scripts/pulumi-up-all.js
const { execSync } = require('child_process');
const path = require('path');
const fs = require('fs');

// Get the Pulumi stack from command line or default to 'dev'
const stack = process.argv[2] || 'dev';

function runPulumiUp(dir) {
  const items = fs.readdirSync(dir);
  
  items.forEach(item => {
    const fullPath = path.join(dir, item);
    const stats = fs.statSync(fullPath);
    
    if (stats.isDirectory()) {
      const pulumiYamlPath = path.join(fullPath, 'Pulumi.yaml');
      
      if (fs.existsSync(pulumiYamlPath)) {
        console.log(`Running pulumi up in ${fullPath}`);
        try {
          execSync(`pulumi up --stack ${stack} --yes`, {
            cwd: fullPath,
            stdio: 'inherit'
          });
        } catch (error) {
          console.error(`Error running pulumi up in ${fullPath}:`, error);
          process.exit(1);
        }
      }
    }
  });
}

// Process infra directory
console.log('Processing infrastructure jobs...');
runPulumiUp('infra');

Here’s how to implement it:

Create a root package.json at the project root level that uses workspaces and shared dependencies.

Move shared devDependencies to the root package.json and keep only project-specific dependencies in the individual job's package.json files.

Create installation and deployment scripts to handle the recursive operations (follow the steps below).

First, simplify your job-specific package.json files to only include necessary dependencies:

{
    "name": "job1",
    "main": "index.ts",
    "version": "1.0.0",
    "dependencies": {
        "@lib/cloudrun": "file:../../lib/job5",
        "@pulumi/docker": "^4.5.8",
        "@pulumi/gcp": "^7.38.0",
        "@pulumi/pulumi": "^3.144.1",
        "@pulumi/random": "^4.16.8",
        "natural-cron": "^1.0.2"
    }
}

Create the scripts directory and add the installation and Pulumi scripts I provided above.

You can now use the following commands:

# Install all dependencies recursively
npm run install-all

# Run Pulumi up for all jobs
npm run pulumi:up-all dev  # or qa/prod for other environments

Here’s some additional recommendations:

Add a .npmrc file at the root to ensure consistent package management:

save-exact=true
legacy-peer-deps=true
workspaces=true

Consider adding these scripts to your justfile for easier execution:

install:
    npm run install-all

deploy ENV:
    npm run pulumi:up-all {{ENV}}

With this approach, shared devDependencies are managed at root level. Each job can still maintain its own specific dependencies, and the scripts provide a consistent way to install dependencies and run Pulumi commands. You can still use mise for version management at the root level.