improve and fix scale issues when building images

builders/lando-v4.js

@@ -291,7 +291,6 @@ module.exports = {
       this.isInteractive = lando.config.isInteractive;
       this.generateCert = lando.generateCert.bind(lando);
       this.network = lando.config.networkBridge;
-      this.project = app.project;
 
       // upstream
       this.user = user;
@@ -414,8 +413,8 @@ module.exports = {
         const leader = file.find(line => line.length > 0).match(/^\s*/)[0].length ?? 0;
         const contents = file.map(line => line.slice(leader)).join('\n');
 
-        // reset file to a path
-        file = path.join(this.context, id ? `${priority}-${id}.sh` : `${priority}-${stage}-${hook}.sh`);
+        // reset file to a path and make executable
+        file = path.join(this.tmpdir, id ? `${priority}-${id}.sh` : `${priority}-${stage}-${hook}.sh`);
         write(file, contents, {forcePosixLineEndings: true});
         fs.chmodSync(file, '755');
       }
@@ -586,9 +585,8 @@ module.exports = {
         }));
       }
 
-      // finally remove the build context
-      fs.rmdirSync(this.context, {force: true, maxRetries: 10, recursive: true});
-      this.debug('removed %o-%o build-context %o', this.project, this.id, this.context);
+      // pass it up
+      await super.destroy();
     }
 
     getBengine() {
@@ -679,7 +677,7 @@ module.exports = {
       contents.push('');
 
       // write to file
-      const npmauthfile = path.join(this.context, 'npmrc');
+      const npmauthfile = path.join(this.tmpdir, 'npmrc');
       write(npmauthfile, contents.join('\n'));
 
       // ensure mount

components/docker-engine.js

@@ -118,19 +118,20 @@ class DockerEngine extends Dockerode {
 
     // extend debugger in appropriate way
     const debug = id ? this.debug.extend(id) : this.debug.extend('docker-engine:build');
-    // collect some args we can merge into promise resolution
-    // @TODO: obscure auth?
-    const args = {command: 'dockerode buildImage', args: {dockerfile, tag, sources}};
-    // create an event emitter we can pass into the promisifier
-    const builder = new EventEmitter();
 
-    // ensure context dir exists
+    // wipe context dir so we get a fresh slate each build
+    fs.removeSync(context, {force: true, maxRetries: 10, recursive: true});
     fs.mkdirSync(context, {recursive: true});
 
     // move other sources into the build context
     for (const source of sources) {
-      fs.copySync(source.source, path.join(context, source.destination));
-      debug('copied %o into build context %o', source.source, path.join(context, source.destination));
+      try {
+        fs.copySync(source.source, path.join(context, source.destination), {dereference: true});
+        debug('copied %o into build context %o', source.source, path.join(context, source.destination));
+      } catch (error) {
+        error.message = `Failed to copy ${source.source} into build context at ${source.destination}!: ${error.message}`;
+        throw error;
+      }
     }
 
     // copy the dockerfile to the correct place
@@ -146,6 +147,12 @@ class DockerEngine extends Dockerode {
       }
     }
 
+    // collect some args we can merge into promise resolution
+    // @TODO: obscure auth?
+    const args = {command: 'dockerode buildImage', args: {dockerfile, tag, sources}};
+    // create an event emitter we can pass into the promisifier
+    const builder = new EventEmitter();
+
     // call the parent
     // @TODO: consider other opts? https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageBuild args?
     debug('building image %o from %o writh build-args %o', tag, context, buildArgs);
@@ -213,6 +220,33 @@ class DockerEngine extends Dockerode {
     // extend debugger in appropriate way
     const debug = id ? this.debug.extend(id) : this.debug.extend('docker-engine:buildx');
 
+    // wipe context dir so we get a fresh slate each build
+    fs.removeSync(context, {force: true, maxRetries: 10, recursive: true});
+    fs.mkdirSync(context, {recursive: true});
+
+    // move sources into the build context if needed
+    for (const source of sources) {
+      try {
+        fs.copySync(source.source, path.join(context, source.destination), {dereference: true});
+        debug('copied %o into build context %o', source.source, path.join(context, source.destination));
+      } catch (error) {
+        error.message = `Failed to copy ${source.source} into build context at ${source.destination}!: ${error.message}`;
+        throw error;
+      }
+    }
+
+    // on windows we want to ensure the build context has linux line endings
+    if (process.platform === 'win32') {
+      for (const file of require('glob').sync(path.join(context, '**/*'), {nodir: true})) {
+        write(file, read(file), {forcePosixLineEndings: true});
+      }
+    }
+
+    // copy the dockerfile to the correct place and reset
+    fs.copySync(dockerfile, path.join(context, 'Dockerfile'));
+    debug('copied Imagefile from %o to %o', dockerfile, path.join(context, 'Dockerfile'));
+    dockerfile = path.join(context, 'Dockerfile');
+
     // build initial buildx command
     const args = {
       command: this.builder,
@@ -278,27 +312,6 @@ class DockerEngine extends Dockerode {
       }
     });
 
-    // ensure context dir exists
-    fs.mkdirSync(context, {recursive: true});
-
-    // move other sources into the build contex
-    // we read/write so we can make sure we are removing windows line endings
-    for (const source of sources) {
-      fs.copySync(source.source, path.join(context, source.destination));
-      debug('copied %o into build context %o', source.source, path.join(context, source.destination));
-    }
-
-    // copy the dockerfile to the correct place
-    fs.copySync(dockerfile, path.join(context, 'Dockerfile'));
-    debug('copied Imagefile from %o to %o', dockerfile, path.join(context, 'Dockerfile'));
-
-    // on windows we want to ensure the build context has linux line endings
-    if (process.platform === 'win32') {
-      for (const file of require('glob').sync(path.join(context, '**/*'), {nodir: true})) {
-        write(file, read(file), {forcePosixLineEndings: true});
-      }
-    }
-
     // debug
     debug('buildxing image %o from %o with build-args', tag, context, buildArgs);
 

components/l337-v4.js

@@ -25,6 +25,7 @@ const toPosixPath = require('../utils/to-posix-path');
 class L337ServiceV4 extends EventEmitter {
   #app
   #data
+  #lando
 
   static debug = require('debug')('@lando/l337-service-v4');
   static bengineConfig = {};
@@ -100,21 +101,23 @@ class L337ServiceV4 extends EventEmitter {
   }
 
   constructor(id, {
-    appRoot = path.join(os.tmpdir(), nanoid(), id),
+    appRoot = path.join(os.tmpdir(), project, 'app', id),
     buildArgs = {},
-    context = path.join(os.tmpdir(), nanoid(), id),
+    context = path.join(os.tmpdir(), project, 'build-contexts', id),
     config = {},
     debug = L337ServiceV4.debug,
     groups = {},
     info = {},
     name = id,
     primary = false,
+    project = app.project,
     sshKeys = [],
     sshSocket = false,
     stages = {},
     states = {},
     tag = nanoid(),
     tlvolumes = {},
+    tmpdir = path.join(os.tmpdir(), project, 'tmp', id),
     type = 'l337',
     user = undefined,
   } = {}, app, lando) {
@@ -128,22 +131,26 @@ class L337ServiceV4 extends EventEmitter {
     this.config = config;
     this.context = context;
     this.debug = debug;
-    this.name = name || id;
+    this.name = name ?? id;
     this.primary = primary;
+    this.project = project;
     this.sshKeys = sshKeys;
     this.sshSocket = sshSocket;
-    this.type = type;
     this.tag = tag;
+    this.tmpdir = tmpdir;
+    this.type = type;
 
-    this.imagefile = path.join(context, 'Imagefile');
+    this.imagefile = path.join(tmpdir, 'Imagefile');
     // @TODO: add needed validation for above things?
     // @TODO: error handling on props?
 
     // makre sure the build context dir exists
     fs.mkdirSync(this.context, {recursive: true});
+    fs.mkdirSync(this.tmpdir, {recursive: true});
 
     // initialize our private data
     this.#app = app;
+    this.#lando = lando;
     this.#data = merge(this.#init(), {groups}, {stages}, {states}, {volumes: Object.keys(tlvolumes)});
 
     // rework info based on whatever is passed in
@@ -466,7 +473,7 @@ class L337ServiceV4 extends EventEmitter {
     if (keys === true) {
       keys = [
         path.join(os.homedir(), '.ssh'),
-        path.resolve(this.context, '..', '..', '..', '..', 'keys'),
+        path.resolve(this.#lando.config.userConfRoot, 'keys'),
       ];
     }
 
@@ -538,7 +545,8 @@ class L337ServiceV4 extends EventEmitter {
       // augment error
       error.context = {imagefile, ...context};
       error.logfile = path.join(context.context ?? os.tmpdir(), `error-${nanoid()}.log`);
-      this.debug('image %o build failed with code %o error %o', context.id, error.code, error);
+      this.debug('image %o build failed with code %o error %o', context.id, error.code ?? 1, error.message);
+      this.debug('%o', error?.stack ?? error);
 
       // inject helpful failing stuff to compose
       this.addComposeData({services: {[context.id]: {
@@ -557,6 +565,14 @@ class L337ServiceV4 extends EventEmitter {
     }
   }
 
+  async destroy() {
+    // remove build contexts and tmp
+    fs.rmSync(this.context, {force: true, maxRetries: 10, recursive: true});
+    fs.rmSync(this.tmpdir, {force: true, maxRetries: 10, recursive: true});
+    this.debug('removed %o-%o build-context %o', this.project, this.id, this.context);
+    this.debug('removed %o-%o tmpdir %o', this.project, this.id, this.tmpdir);
+  }
+
   generateBuildContext() {
     // get dockerfile validator
     // const {validate} = require('dockerfile-utils');

examples/mounts/.lando.yml

@@ -3,6 +3,9 @@ services:
   web1:
     api: 4
     image: nginxinc/nginx-unprivileged:1.26.1
+    app-mount: false
+    overrides:
+      working_dir: /app
     user: nginx
     ports:
       - 8080/http
@@ -21,6 +24,16 @@ services:
           hello
           there!
 
+      # copy source file to target
+      - source: ./other-deps/that-go-here/dep1
+        target: /deps/dep1
+        type: copy
+
+      # copy app files into webroot
+      - source: ./
+        target: /usr/share/nginx/html
+        type: copy
+
       # copy contents to target during config stage of image build
       - target: /etc/lando/config/anakin
         type: copy
@@ -49,15 +62,34 @@ services:
           - "other-deps/that-go-here"
 
     image: node:18
-    command: bash -c "pwd && npm run dev"
+    command: npm run dev
     ports:
       - 3000/http
     user: node
     build:
       app: |
-        pwd
         npm install
 
+  web3:
+    api: 4
+    app-mount: false
+    overrides:
+      working_dir: /app
+    mounts:
+      - source: ./
+        destination: /app
+        type: copy
+
+    image: node:18
+    command: sleep infinity
+    # command: npm run dev
+    # ports:
+    #   - 3000/http
+    user: node
+    # build:
+    #   image: |ls -
+    #     npm install
+
 plugins:
   "@lando/core": "../.."
   "@lando/healthcheck": "../../plugins/healthcheck"

hooks/app-add-v4-services.js

@@ -52,8 +52,10 @@ module.exports = async (app, lando) => {
         context: path.join(app.v4._dir, 'build-contexts', config.name),
         debug: app.v4._debugShim,
         info,
+        project: app.project,
         tag: `${_.get(lando, 'product', 'lando')}/${app.name}-${app.id}-${config.name}:latest`,
         tlvolumes: app.config.volumes,
+        tmpdir: path.join(app.v4._dir, 'tmp', config.name),
       },
       ...config,
     }, app, lando);

utils/normalize-mounts.js

@@ -8,7 +8,7 @@ const toPosixPath = require('./to-posix-path');
 const write = require('./write-file');
 const uniq = require('lodash/uniq');
 
-module.exports = (mounts, {appRoot, context, normalizeVolumes, _data}) => {
+module.exports = (mounts, {_data, appRoot, normalizeVolumes, tmpdir}) => {
   return mounts.map(mount => {
     // if mount is a single string then assume its a bind mount and normalize
     if (typeof mount === 'string' && toPosixPath(mount).split(':').length > 1) {
@@ -35,7 +35,7 @@ module.exports = (mounts, {appRoot, context, normalizeVolumes, _data}) => {
     // note that this means source wins over content|contents if both are present
     if ((mount.contents || mount.content) && !mount.source) {
       // dump contents to file and set source to it
-      mount.source = path.join(context, mount.target.split(path.sep).filter(part => part !== '').join('-'));
+      mount.source = path.join(tmpdir, mount.target.split(path.sep).filter(part => part !== '').join('-'));
       fs.mkdirSync(path.dirname(mount.source), {force: true, maxRetries: 10, recursive: true});
       fs.rmSync(mount.source, {force: true, maxRetries: 10, recursive: true});
       write(mount.source, mount.contents ?? mount.content);