diff --git a/examples/build/build_buildkit.js b/examples/build/build_buildkit.js new file mode 100644 index 0000000..d19cf10 --- /dev/null +++ b/examples/build/build_buildkit.js @@ -0,0 +1,37 @@ +var Docker = require('../../lib/docker'), + tar = require('tar-fs'); + +var docker = new Docker({ + socketPath: '/var/run/docker.sock' +}); + +var tarStream = tar.pack(process.cwd()); + +// BuildKit v2 builds return base64-encoded protobuf logs +// Use docker.followProgress() to decode them automatically +// (This works the same as docker.modem.followProgress but decodes BuildKit output) +docker.buildImage(tarStream, { + t: 'myimage:buildkit', + version: '2' // Enable BuildKit +}, function(error, stream) { + if (error) { + return console.error(error); + } + + // docker.followProgress works with both regular and BuildKit builds + docker.followProgress(stream, + function onFinished(err, result) { + if (err) { + console.error('Build failed:', err); + } else { + console.log('Build completed successfully!'); + } + }, + function onProgress(event) { + // Each event is already decoded and formatted + if (event.stream) { + process.stdout.write(event.stream); + } + } + ); +}); diff --git a/lib/buildkit.js b/lib/buildkit.js new file mode 100644 index 0000000..b9c152e --- /dev/null +++ b/lib/buildkit.js @@ -0,0 +1,242 @@ +var protobuf = require("protobufjs"); +var path = require("path"); + +// Constants +var BUILDKIT_TRACE_ID = "moby.buildkit.trace"; +var BUILDKIT_IMAGE_ID = "moby.image.id"; +var PROTO_TYPE = "moby.buildkit.v1.StatusResponse"; +var ENCODING_UTF8 = "utf8"; +var ENCODING_BASE64 = "base64"; + +var StatusResponse; + +// Load the protobuf schema +function loadProto() { + if (StatusResponse) return StatusResponse; + + var root = protobuf.loadSync( + path.resolve(__dirname, "proto", "buildkit_status.proto") + ); + StatusResponse = root.lookupType(PROTO_TYPE); + return StatusResponse; +} + +/** + * Decodes a BuildKit trace message + * @param {string} base64Data - Base64-encoded protobuf data from aux field + * @returns {Object} Decoded status response with vertexes, logs, etc. + */ +function decodeBuildKitStatus(base64Data) { + var StatusResponse = loadProto(); + + // Handle empty messages + if (!base64Data || base64Data.length === 0) { + return { + vertexes: [], + statuses: [], + logs: [], + warnings: [] + }; + } + + var buffer = Buffer.from(base64Data, ENCODING_BASE64); + var message = StatusResponse.decode(buffer); + return StatusResponse.toObject(message, { + longs: String, + enums: String, + bytes: String, + defaults: true + }); +} + +/** + * Formats BuildKit status into human-readable text + * @param {Object} status - Decoded status response + * @returns {string[]} Array of human-readable log lines + */ +function formatBuildKitStatus(status) { + var lines = []; + + // Process vertexes (build steps) + if (status.vertexes && status.vertexes.length > 0) { + status.vertexes.forEach(function(vertex) { + if (vertex.name && vertex.started && !vertex.completed) { + lines.push("[" + vertex.digest.substring(0, 12) + "] " + vertex.name); + } + if (vertex.error) { + lines.push("ERROR: " + vertex.error); + } + if (vertex.completed && vertex.cached) { + lines.push("CACHED: " + vertex.name); + } + }); + } + + // Process logs (command output) + if (status.logs && status.logs.length > 0) { + status.logs.forEach(function(log) { + var msg = Buffer.from(log.msg).toString(ENCODING_UTF8); + if (msg.trim()) { + lines.push(msg.trimEnd()); + } + }); + } + + // Process status updates (progress) + if (status.statuses && status.statuses.length > 0) { + status.statuses.forEach(function(s) { + if (s.name && s.total > 0) { + var percent = Math.floor((s.current / s.total) * 100); + lines.push(s.name + ": " + percent + "% (" + s.current + "/" + s.total + ")"); + } + }); + } + + // Process warnings + if (status.warnings && status.warnings.length > 0) { + status.warnings.forEach(function(warning) { + var msg = Buffer.from(warning.short).toString(ENCODING_UTF8); + lines.push("WARNING: " + msg); + }); + } + + return lines; +} + +/** + * Parse a BuildKit stream line and extract human-readable logs + * @param {string} line - JSON line from build stream + * @returns {Object} { isBuildKit: boolean, logs: string[], raw: Object } + */ +function parseBuildKitLine(line) { + try { + var json = JSON.parse(line); + + // Check if it's a BuildKit trace message + if (json.id === BUILDKIT_TRACE_ID && json.aux !== undefined) { + var status = decodeBuildKitStatus(json.aux); + var logs = formatBuildKitStatus(status); + + return { + isBuildKit: true, + logs: logs, + raw: status + }; + } + + // Check if it's the final image ID + if (json.id === BUILDKIT_IMAGE_ID && json.aux && json.aux.ID) { + return { + isBuildKit: true, + logs: ["Built image: " + json.aux.ID], + raw: json.aux + }; + } + + // Not a BuildKit message + return { + isBuildKit: false, + logs: [], + raw: json + }; + } catch (e) { + return { + isBuildKit: false, + logs: [], + raw: null, + error: e.message + }; + } +} + +/** + * Follow progress of a stream, automatically handling both BuildKit and regular output. + * This provides the same ergonomics as modem.followProgress but decodes BuildKit logs. + * + * @param {Stream} stream - Stream from buildImage(), pull(), push(), etc. + * @param {Function} onFinished - Called when stream ends: (err, output) => void + * @param {Function} onProgress - Called for each log event: (event) => void + * @returns {void} + */ +function followProgress(stream, onFinished, onProgress) { + var buffer = ''; + var output = []; + var finished = false; + + stream.on('data', onStreamEvent); + stream.on('error', onStreamError); + stream.on('end', onStreamEnd); + stream.on('close', onStreamEnd); + + function onStreamEvent(data) { + buffer += data.toString(); + + // Process complete lines + var lines = buffer.split('\n'); + buffer = lines.pop(); // Save incomplete line + + lines.forEach(function(line) { + if (!line.trim()) return; + + processLine(line); + }); + } + + function processLine(line) { + try { + // Try to parse as BuildKit or regular Docker output + var result = parseBuildKitLine(line); + + if (result.isBuildKit) { + // BuildKit message - create events from decoded logs + result.logs.forEach(function(log) { + var event = { stream: log + '\n' }; + output.push(event); + if (onProgress) onProgress(event); + }); + } else if (result.raw) { + // Regular Docker message + output.push(result.raw); + if (onProgress) onProgress(result.raw); + } + } catch (e) { + // If parsing fails, try plain JSON + try { + var json = JSON.parse(line); + output.push(json); + if (onProgress) onProgress(json); + } catch (e2) { + // Ignore parse errors + } + } + } + + function onStreamError(err) { + finished = true; + stream.removeListener('data', onStreamEvent); + stream.removeListener('error', onStreamError); + stream.removeListener('end', onStreamEnd); + stream.removeListener('close', onStreamEnd); + if (onFinished) onFinished(err, output); + } + + function onStreamEnd() { + if (finished) return; + finished = true; + + // Process any remaining data in buffer + if (buffer.trim()) { + processLine(buffer); + } + + stream.removeListener('data', onStreamEvent); + stream.removeListener('error', onStreamError); + stream.removeListener('end', onStreamEnd); + stream.removeListener('close', onStreamEnd); + if (onFinished) onFinished(null, output); + } +} + +module.exports = { + followProgress: followProgress +}; diff --git a/lib/docker.js b/lib/docker.js index e3739dd..4951f8d 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -335,6 +335,30 @@ Docker.prototype.buildImage = function(file, opts, callback) { } }; +/** + * Follow progress of a stream operation (build, pull, push, etc.) with automatic + * BuildKit decoding. + * + * This method works identically to docker.modem.followProgress() but additionally + * decodes BuildKit v2 build output. BuildKit emits base64-encoded protobuf messages + * which this method transparently decodes into human-readable log events. + * + * Use this instead of docker.modem.followProgress() when: + * - You're using BuildKit builds (version: "2") + * - You want a single API that handles both regular and BuildKit output + * + * For non-BuildKit streams (pull, push, regular builds), behavior is identical + * to docker.modem.followProgress(). + * + * @param {Stream} stream - Stream from buildImage(), pull(), push(), etc. + * @param {Function} onFinished - Called when stream ends: (err, output) => void + * @param {Function} onProgress - Optional callback for each event: (event) => void + */ +Docker.prototype.followProgress = function(stream, onFinished, onProgress) { + var buildkit = require('./buildkit'); + return buildkit.followProgress(stream, onFinished, onProgress); +}; + /** * Fetches a Container by ID * @param {String} id Container's ID diff --git a/lib/proto/buildkit_status.proto b/lib/proto/buildkit_status.proto new file mode 100644 index 0000000..b5ca5b1 --- /dev/null +++ b/lib/proto/buildkit_status.proto @@ -0,0 +1,81 @@ +syntax = "proto3"; + +package moby.buildkit.v1; + +// Minimal definitions for decoding BuildKit status messages +// Based on https://github.com/moby/buildkit/blob/master/api/services/control/control.proto +// Related to https://github.com/moby/buildkit/blob/master/solver/pb/ops.proto (vertices map to the solver op DAG) + +message StatusResponse { + repeated Vertex vertexes = 1; + repeated VertexStatus statuses = 2; + repeated VertexLog logs = 3; + repeated VertexWarning warnings = 4; +} + +message Vertex { + string digest = 1; + repeated string inputs = 2; + string name = 3; + bool cached = 4; + Timestamp started = 5; + Timestamp completed = 6; + string error = 7; + ProgressGroup progressGroup = 8; +} + +message VertexStatus { + string ID = 1; + string vertex = 2; + string name = 3; + int64 current = 4; + int64 total = 5; + Timestamp timestamp = 6; + Timestamp started = 7; + Timestamp completed = 8; +} + +message VertexLog { + string vertex = 1; + Timestamp timestamp = 2; + int64 stream = 3; + bytes msg = 4; +} + +message VertexWarning { + string vertex = 1; + int64 level = 2; + bytes short = 3; + repeated bytes detail = 4; + string url = 5; + SourceInfo info = 6; + repeated Range ranges = 7; +} + +message ProgressGroup { + string id = 1; + string name = 2; + bool weak = 3; +} + +// Simplified Timestamp to match google.protobuf.Timestamp wire format +message Timestamp { + int64 seconds = 1; + int32 nanos = 2; +} + +message SourceInfo { + string filename = 1; + bytes data = 2; + // definition and language fields omitted - not needed for log decoding +} + +message Range { + Position start = 1; + Position end = 2; +} + +message Position { + int32 line = 1; + int32 character = 2; +} diff --git a/test/buildkit_test.js b/test/buildkit_test.js new file mode 100644 index 0000000..04942b0 --- /dev/null +++ b/test/buildkit_test.js @@ -0,0 +1,238 @@ +var expect = require("chai").expect; +var Docker = require("../lib/docker"); +var stream = require("stream"); +var PassThrough = stream.PassThrough; + +describe("#followProgress", function() { + var docker = new Docker(); + + describe("stream handling", function() { + it("should handle incomplete lines across chunks", function(done) { + var mockStream = new PassThrough(); + var results = []; + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output.length).to.equal(2); + expect(output[0].stream).to.include("Step 1"); + expect(output[1].stream).to.include("Step 2"); + done(); + }, + function onProgress(event) { + results.push(event); + } + ); + + // Simulate chunks that split lines awkwardly + mockStream.write('{"stream":"Step 1/3 : FROM'); + mockStream.write(' alpine\\n"}\n{"stream":"'); + mockStream.write('Step 2/3 : RUN echo test\\n"}\n'); + mockStream.end(); + }); + + it("should handle chunk ending mid-line", function(done) { + var mockStream = new PassThrough(); + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output.length).to.equal(2); + expect(output[0].stream).to.include("Complete line"); + expect(output[1].stream).to.include("Incomplete line"); + done(); + } + ); + + // First chunk ends in middle of a line (no newline) + mockStream.write('{"stream":"Complete line\\n"}\n{"stream":"Incompl'); + // Second chunk completes it + mockStream.write('ete line\\n"}\n'); + mockStream.end(); + }); + + it("should handle BuildKit messages split across chunks", function(done) { + var mockStream = new PassThrough(); + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output).to.be.an("array"); + done(); + } + ); + + // Split a BuildKit trace message across chunks + var buildkitMsg = '{"id":"moby.buildkit.trace","aux":""}\n'; + var mid = Math.floor(buildkitMsg.length / 2); + + mockStream.write(buildkitMsg.substring(0, mid)); + mockStream.write(buildkitMsg.substring(mid)); + mockStream.end(); + }); + + it("should decode real BuildKit base64 protobuf messages", function(done) { + var mockStream = new PassThrough(); + var receivedEvents = []; + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output.length).to.be.greaterThan(0); + // Should have decoded the base64 protobuf successfully + var hasDecodedLog = receivedEvents.some(function(e) { + return e.stream && e.stream.includes("internal"); + }); + expect(hasDecodedLog).to.be.true; + done(); + }, + function onProgress(event) { + receivedEvents.push(event); + } + ); + + // Real BuildKit message with actual base64 protobuf data + // Decodes to: "[internal] load remote build context" + var buildkitMsg = '{"id":"moby.buildkit.trace","aux":"Cm8KR3NoYTI1NjpkMDZmYWJlMGZmMTMzZTVhN2Q4ODE2Yjg3ZTdjYmE2ZjUwZWI3ZDM0NWY3N2EyY2Y5M2Y4NmI1OWFiZWFiNWNhGiRbaW50ZXJuYWxdIGxvYWQgcmVtb3RlIGJ1aWxkIGNvbnRleHQKfApHc2hhMjU2OmQwNmZhYmUwZmYxMzNlNWE3ZDg4MTZiODdlN2NiYTZmNTBlYjdkMzQ1Zjc3YTJjZjkzZjg2YjU5YWJlYWI1Y2EaJFtpbnRlcm5hbF0gbG9hZCByZW1vdGUgYnVpbGQgY29udGV4dCoLCL6hz8sGEJuPn2E="}\n'; + + mockStream.write(buildkitMsg); + mockStream.end(); + }); + + it("should ignore empty lines", function(done) { + var mockStream = new PassThrough(); + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + // Should only have 2 events, empty lines ignored + expect(output.length).to.equal(2); + done(); + } + ); + + mockStream.write('{"stream":"Line 1\\n"}\n'); + mockStream.write('\n'); // Empty line + mockStream.write('\n'); // Another empty line + mockStream.write('{"stream":"Line 2\\n"}\n'); + mockStream.end(); + }); + + it("should process final buffered data on stream end", function(done) { + var mockStream = new PassThrough(); + + docker.followProgress(mockStream, + function onFinished(err, output) { + expect(err).to.be.null; + // Should process the line that didn't end with \n + expect(output.length).to.equal(1); + expect(output[0].stream).to.include("Final"); + done(); + } + ); + + // Send a line without trailing newline + mockStream.write('{"stream":"Final line\\n"}'); + mockStream.end(); // Should trigger processing of buffered data + }); + + it("should work without onProgress callback", function(done) { + var mockStream = new PassThrough(); + + docker.followProgress(mockStream, function(err, output) { + expect(err).to.be.null; + expect(output).to.be.an("array"); + expect(output.length).to.equal(2); + done(); + }); + + mockStream.write('{"stream":"Line 1\\n"}\n'); + mockStream.write('{"stream":"Line 2\\n"}\n'); + mockStream.end(); + }); + }); + + describe("integration", function() { + it("should follow BuildKit build progress", function(done) { + this.timeout(60000); + var randomId = "buildkit-test-" + Date.now(); + + docker.buildImage( + { + context: __dirname, + src: ["buildkit.Dockerfile"] + }, + { + dockerfile: "buildkit.Dockerfile", + version: "2", + t: randomId, + }, + function(err, stream) { + expect(err).to.be.null; + expect(stream).to.be.ok; + + var progressEvents = []; + + docker.followProgress(stream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output).to.be.an("array"); + expect(output.length).to.be.greaterThan(0); + + console.log("\n Total events:", output.length); + console.log(" Progress callbacks:", progressEvents.length); + + // Clean up + docker.getImage(randomId).remove(function() { + done(); + }); + }, + function onProgress(event) { + progressEvents.push(event); + if (event.stream) { + console.log(" Progress:", event.stream.trim().substring(0, 60)); + } + } + ); + } + ); + }); + + it("should work with regular builds too", function(done) { + this.timeout(60000); + var randomId = "regular-test-" + Date.now(); + + docker.buildImage( + { + context: __dirname, + src: ["buildkit.Dockerfile"] + }, + { + dockerfile: "buildkit.Dockerfile", + // No version: "2" - regular build + t: randomId, + }, + function(err, stream) { + expect(err).to.be.null; + + docker.followProgress(stream, + function onFinished(err, output) { + expect(err).to.be.null; + expect(output).to.be.an("array"); + + console.log("\n Regular build events:", output.length); + + docker.getImage(randomId).remove(function() { + done(); + }); + }, + function onProgress(event) { + // Regular builds have different event format + expect(event).to.be.an("object"); + } + ); + } + ); + }); + }); +});