buildkit proto followProgress support

examples/build/build_buildkit.js

@@ -0,0 +1,37 @@
+var Docker = require('../../lib/docker'),
+  tar = require('tar-fs');
+
+var docker = new Docker({
+  socketPath: '/var/run/docker.sock'
+});
+
+var tarStream = tar.pack(process.cwd());
+
+// BuildKit v2 builds return base64-encoded protobuf logs
+// Use docker.followProgress() to decode them automatically
+// (This works the same as docker.modem.followProgress but decodes BuildKit output)
+docker.buildImage(tarStream, {
+  t: 'myimage:buildkit',
+  version: '2'  // Enable BuildKit
+}, function(error, stream) {
+  if (error) {
+    return console.error(error);
+  }
+
+  // docker.followProgress works with both regular and BuildKit builds
+  docker.followProgress(stream,
+    function onFinished(err, result) {
+      if (err) {
+        console.error('Build failed:', err);
+      } else {
+        console.log('Build completed successfully!');
+      }
+    },
+    function onProgress(event) {
+      // Each event is already decoded and formatted
+      if (event.stream) {
+        process.stdout.write(event.stream);
+      }
+    }
+  );
+});

lib/buildkit.js

@@ -0,0 +1,242 @@
+var protobuf = require("protobufjs");
+var path = require("path");
+
+// Constants
+var BUILDKIT_TRACE_ID = "moby.buildkit.trace";
+var BUILDKIT_IMAGE_ID = "moby.image.id";
+var PROTO_TYPE = "moby.buildkit.v1.StatusResponse";
+var ENCODING_UTF8 = "utf8";
+var ENCODING_BASE64 = "base64";
+
+var StatusResponse;
+
+// Load the protobuf schema
+function loadProto() {
+  if (StatusResponse) return StatusResponse;
+
+  var root = protobuf.loadSync(
+    path.resolve(__dirname, "proto", "buildkit_status.proto")
+  );
+  StatusResponse = root.lookupType(PROTO_TYPE);
+  return StatusResponse;
+}
+
+/**
+ * Decodes a BuildKit trace message
+ * @param {string} base64Data - Base64-encoded protobuf data from aux field
+ * @returns {Object} Decoded status response with vertexes, logs, etc.
+ */
+function decodeBuildKitStatus(base64Data) {
+  var StatusResponse = loadProto();
+
+  // Handle empty messages
+  if (!base64Data || base64Data.length === 0) {
+    return {
+      vertexes: [],
+      statuses: [],
+      logs: [],
+      warnings: []
+    };
+  }
+
+  var buffer = Buffer.from(base64Data, ENCODING_BASE64);
+  var message = StatusResponse.decode(buffer);
+  return StatusResponse.toObject(message, {
+    longs: String,
+    enums: String,
+    bytes: String,
+    defaults: true
+  });
+}
+
+/**
+ * Formats BuildKit status into human-readable text
+ * @param {Object} status - Decoded status response
+ * @returns {string[]} Array of human-readable log lines
+ */
+function formatBuildKitStatus(status) {
+  var lines = [];
+
+  // Process vertexes (build steps)
+  if (status.vertexes && status.vertexes.length > 0) {
+    status.vertexes.forEach(function(vertex) {
+      if (vertex.name && vertex.started && !vertex.completed) {
+        lines.push("[" + vertex.digest.substring(0, 12) + "] " + vertex.name);
+      }
+      if (vertex.error) {
+        lines.push("ERROR: " + vertex.error);
+      }
+      if (vertex.completed && vertex.cached) {
+        lines.push("CACHED: " + vertex.name);
+      }
+    });
+  }
+
+  // Process logs (command output)
+  if (status.logs && status.logs.length > 0) {
+    status.logs.forEach(function(log) {
+      var msg = Buffer.from(log.msg).toString(ENCODING_UTF8);
+      if (msg.trim()) {
+        lines.push(msg.trimEnd());
+      }
+    });
+  }
+
+  // Process status updates (progress)
+  if (status.statuses && status.statuses.length > 0) {
+    status.statuses.forEach(function(s) {
+      if (s.name && s.total > 0) {
+        var percent = Math.floor((s.current / s.total) * 100);
+        lines.push(s.name + ": " + percent + "% (" + s.current + "/" + s.total + ")");
+      }
+    });
+  }
+
+  // Process warnings
+  if (status.warnings && status.warnings.length > 0) {
+    status.warnings.forEach(function(warning) {
+      var msg = Buffer.from(warning.short).toString(ENCODING_UTF8);
+      lines.push("WARNING: " + msg);
+    });
+  }
+
+  return lines;
+}
+
+/**
+ * Parse a BuildKit stream line and extract human-readable logs
+ * @param {string} line - JSON line from build stream
+ * @returns {Object} { isBuildKit: boolean, logs: string[], raw: Object }
+ */
+function parseBuildKitLine(line) {
+  try {
+    var json = JSON.parse(line);
+
+    // Check if it's a BuildKit trace message
+    if (json.id === BUILDKIT_TRACE_ID && json.aux !== undefined) {
+      var status = decodeBuildKitStatus(json.aux);
+      var logs = formatBuildKitStatus(status);
+
+      return {
+        isBuildKit: true,
+        logs: logs,
+        raw: status
+      };
+    }
+
+    // Check if it's the final image ID
+    if (json.id === BUILDKIT_IMAGE_ID && json.aux && json.aux.ID) {
+      return {
+        isBuildKit: true,
+        logs: ["Built image: " + json.aux.ID],
+        raw: json.aux
+      };
+    }
+
+    // Not a BuildKit message
+    return {
+      isBuildKit: false,
+      logs: [],
+      raw: json
+    };
+  } catch (e) {
+    return {
+      isBuildKit: false,
+      logs: [],
+      raw: null,
+      error: e.message
+    };
+  }
+}
+
+/**
+ * Follow progress of a stream, automatically handling both BuildKit and regular output.
+ * This provides the same ergonomics as modem.followProgress but decodes BuildKit logs.
+ * 
+ * @param {Stream} stream - Stream from buildImage(), pull(), push(), etc.
+ * @param {Function} onFinished - Called when stream ends: (err, output) => void
+ * @param {Function} onProgress - Called for each log event: (event) => void
+ * @returns {void}
+ */
+function followProgress(stream, onFinished, onProgress) {
+  var buffer = '';
+  var output = [];
+  var finished = false;
+
+  stream.on('data', onStreamEvent);
+  stream.on('error', onStreamError);
+  stream.on('end', onStreamEnd);
+  stream.on('close', onStreamEnd);
+
+  function onStreamEvent(data) {
+    buffer += data.toString();
+
+    // Process complete lines
+    var lines = buffer.split('\n');
+    buffer = lines.pop(); // Save incomplete line
+
+    lines.forEach(function(line) {
+      if (!line.trim()) return;
+
+      processLine(line);
+    });
+  }
+
+  function processLine(line) {
+    try {
+      // Try to parse as BuildKit or regular Docker output
+      var result = parseBuildKitLine(line);
+
+      if (result.isBuildKit) {
+        // BuildKit message - create events from decoded logs
+        result.logs.forEach(function(log) {
+          var event = { stream: log + '\n' };
+          output.push(event);
+          if (onProgress) onProgress(event);
+        });
+      } else if (result.raw) {
+        // Regular Docker message
+        output.push(result.raw);
+        if (onProgress) onProgress(result.raw);
+      }
+    } catch (e) {
+      // If parsing fails, try plain JSON
+      try {
+        var json = JSON.parse(line);
+        output.push(json);
+        if (onProgress) onProgress(json);
+      } catch (e2) {
+        // Ignore parse errors
+      }
+    }
+  }
+
+  function onStreamError(err) {
+    finished = true;
+    stream.removeListener('data', onStreamEvent);
+    stream.removeListener('error', onStreamError);
+    stream.removeListener('end', onStreamEnd);
+    stream.removeListener('close', onStreamEnd);
+    if (onFinished) onFinished(err, output);
+  }
+
+  function onStreamEnd() {
+    if (finished) return;
+    finished = true;
+
+    // Process any remaining data in buffer
+    if (buffer.trim()) {
+      processLine(buffer);
+    }
+
+    stream.removeListener('data', onStreamEvent);
+    stream.removeListener('error', onStreamError);
+    stream.removeListener('end', onStreamEnd);
+    stream.removeListener('close', onStreamEnd);
+    if (onFinished) onFinished(null, output);
+  }
+}
+
+module.exports = {
+  followProgress: followProgress
+};

lib/docker.js

@@ -335,6 +335,30 @@ Docker.prototype.buildImage = function(file, opts, callback) {
   }
 };
 
+/**
+ * Follow progress of a stream operation (build, pull, push, etc.) with automatic
+ * BuildKit decoding.
+ * 
+ * This method works identically to docker.modem.followProgress() but additionally
+ * decodes BuildKit v2 build output. BuildKit emits base64-encoded protobuf messages
+ * which this method transparently decodes into human-readable log events.
+ * 
+ * Use this instead of docker.modem.followProgress() when:
+ * - You're using BuildKit builds (version: "2")
+ * - You want a single API that handles both regular and BuildKit output
+ * 
+ * For non-BuildKit streams (pull, push, regular builds), behavior is identical
+ * to docker.modem.followProgress().
+ * 
+ * @param {Stream} stream - Stream from buildImage(), pull(), push(), etc.
+ * @param {Function} onFinished - Called when stream ends: (err, output) => void
+ * @param {Function} onProgress - Optional callback for each event: (event) => void
+ */
+Docker.prototype.followProgress = function(stream, onFinished, onProgress) {
+  var buildkit = require('./buildkit');
+  return buildkit.followProgress(stream, onFinished, onProgress);
+};
+
 /**
  * Fetches a Container by ID
  * @param {String} id Container's ID

lib/proto/buildkit_status.proto

@@ -0,0 +1,81 @@
+syntax = "proto3";
+
+package moby.buildkit.v1;
+
+// Minimal definitions for decoding BuildKit status messages
+// Based on https://github.com/moby/buildkit/blob/master/api/services/control/control.proto
+// Related to https://github.com/moby/buildkit/blob/master/solver/pb/ops.proto (vertices map to the solver op DAG)
+
+message StatusResponse {
+  repeated Vertex vertexes = 1;
+  repeated VertexStatus statuses = 2;
+  repeated VertexLog logs = 3;
+  repeated VertexWarning warnings = 4;
+}
+
+message Vertex {
+  string digest = 1;
+  repeated string inputs = 2;
+  string name = 3;
+  bool cached = 4;
+  Timestamp started = 5;
+  Timestamp completed = 6;
+  string error = 7;
+  ProgressGroup progressGroup = 8;
+}
+
+message VertexStatus {
+  string ID = 1;
+  string vertex = 2;
+  string name = 3;
+  int64 current = 4;
+  int64 total = 5;
+  Timestamp timestamp = 6;
+  Timestamp started = 7;
+  Timestamp completed = 8;
+}
+
+message VertexLog {
+  string vertex = 1;
+  Timestamp timestamp = 2;
+  int64 stream = 3;
+  bytes msg = 4;
+}
+
+message VertexWarning {
+  string vertex = 1;
+  int64 level = 2;
+  bytes short = 3;
+  repeated bytes detail = 4;
+  string url = 5;
+  SourceInfo info = 6;
+  repeated Range ranges = 7;
+}
+
+message ProgressGroup {
+  string id = 1;
+  string name = 2;
+  bool weak = 3;
+}
+
+// Simplified Timestamp to match google.protobuf.Timestamp wire format
+message Timestamp {
+  int64 seconds = 1;
+  int32 nanos = 2;
+}
+
+message SourceInfo {
+  string filename = 1;
+  bytes data = 2;
+  // definition and language fields omitted - not needed for log decoding
+}
+
+message Range {
+  Position start = 1;
+  Position end = 2;
+}
+
+message Position {
+  int32 line = 1;
+  int32 character = 2;
+}