diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DiskSpdProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DiskSpdProfileTests.cs index 8ed6b43992..d9ecc814c5 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DiskSpdProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DiskSpdProfileTests.cs @@ -62,10 +62,154 @@ public async Task DiskSpdWorkloadProfileInstallsTheExpectedDependenciesOnWindows } [Test] - [TestCase("PERF-IO-DISKSPD.json")] + [TestCase("PERF-IO-DISKSPD-RANDWRITE.json", TestName = "DiskSpdRandomWriteProfileExecutes")] + public async Task DiskSpdRandomWriteWorkloadProfileExecutes(string profile) + { + IEnumerable expectedCommands = new List + { + // Disk fill (2 disks) + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + // Random write tests + $"-c496G -b4K -r4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext", + $"-c496G -b1024K -r4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext" + }; + + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.SetupDisks(withUnformatted: false); + this.mockFixture.SetupPackage("diskspd", expectedFiles: $@"win-x64\diskspd.exe"); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("diskspd", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_DiskSpd.txt")); + } + + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-DISKSPD-SEQWRITE.json", TestName = "DiskSpdSequentialWriteProfileExecutes")] + public async Task DiskSpdSequentialWriteWorkloadProfileExecutes(string profile) + { + IEnumerable expectedCommands = new List + { + // Disk fill (2 disks) + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + // Sequential write tests + $"-c496G -b4K -si4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext", + $"-c496G -b1024K -si4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext" + }; + + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.SetupDisks(withUnformatted: false); + this.mockFixture.SetupPackage("diskspd", expectedFiles: $@"win-x64\diskspd.exe"); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("diskspd", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_DiskSpd.txt")); + } + + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-DISKSPD-RANDREAD.json", TestName = "DiskSpdRandomReadProfileExecutes")] + public async Task DiskSpdRandomReadWorkloadProfileExecutes(string profile) + { + IEnumerable expectedCommands = new List + { + // Disk fill (2 disks) + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + // Random read tests + $"-c496G -b4K -r4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext", + $"-c496G -b1024K -r4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext" + }; + + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.SetupDisks(withUnformatted: false); + this.mockFixture.SetupPackage("diskspd", expectedFiles: $@"win-x64\diskspd.exe"); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("diskspd", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_DiskSpd.txt")); + } + + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-DISKSPD-SEQREAD.json", TestName = "DiskSpdSequentialReadProfileExecutes")] + public async Task DiskSpdSequentialReadWorkloadProfileExecutes(string profile) + { + IEnumerable expectedCommands = new List + { + // Disk fill (2 disks) + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + // Sequential read tests + $"-c496G -b4K -si4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext", + $"-c496G -b1024K -si4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext" + }; + + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.SetupDisks(withUnformatted: false); + this.mockFixture.SetupPackage("diskspd", expectedFiles: $@"win-x64\diskspd.exe"); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("diskspd", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_DiskSpd.txt")); + } + + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-DISKSPD-FUNCTIONAL.json", TestName = "DiskSpdFunctionalProfileExecutes")] public async Task DiskSpdWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatform(string profile) { - IEnumerable expectedCommands = DiskSpdProfileTests.GetDiskSpdStressProfileExpectedCommands(); + IEnumerable expectedCommands = DiskSpdProfileTests.GetDiskSpdStressProfileExpectedCommands(functional: true); // Setup the expectations for the workload // - Disks are formatted and ready @@ -117,9 +261,9 @@ public void DiskSpdWorkloadProfileActionsWillNotBeExecutedIfTheWorkloadPackageDo } } - private static IEnumerable GetDiskSpdStressProfileExpectedCommands() + private static IEnumerable GetDiskSpdStressProfileExpectedCommands(bool functional = false) { - return new List + var commands = new List { // Given the test setup created 2 remote disks, we will perform a disk fill on both individually $"-c500G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", @@ -155,6 +299,28 @@ private static IEnumerable GetDiskSpdStressProfileExpectedCommands() $"-c496G -b16K -si4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext", $"-c496G -b1024K -si4K -t[0-9]+ -o[0-9]+ -w0 -d300 -Suw -W15 -D -L -Rtext" }; + + // For functional testing profile, return only the representative subset (6 actions total) + if (functional) + { + return new List + { + // Disk fill (2 disks) + commands[0], + commands[1], + // Random write (small and large block) + commands[2], // 4k + commands[6], // 1024k + // Sequential write (small block) + commands[7], // 4k + // Random read (small block) + commands[12], // 4k + // Sequential read (small block) + commands[17] // 4k + }; + } + + return commands; } } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/FioProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/FioProfileTests.cs index 80ee101854..90992fbea5 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/FioProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/FioProfileTests.cs @@ -84,19 +84,23 @@ public async Task FioWorkloadProfileInstallsTheExpectedDependenciesOnUnixPlatfor } [Test] - [TestCase("PERF-IO-FIO.json")] - public async Task FioWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatform(string profile) + [TestCase("PERF-IO-FIO-RANDWRITE.json", PlatformID.Win32NT, TestName = "FioRandomWriteProfileExecutesOnWindows")] + [TestCase("PERF-IO-FIO-RANDWRITE.json", PlatformID.Unix, TestName = "FioRandomWriteProfileExecutesOnUnix")] + public async Task FioRandomWriteWorkloadProfileExecutes(string profile, PlatformID platform) { - IEnumerable expectedCommands = FioProfileTests.GetFioProfileExpectedCommands(PlatformID.Win32NT); + string expectedIoEngine = platform == PlatformID.Win32NT ? "windowsaio" : "libaio"; + IEnumerable expectedCommands = new List + { + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread ", + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread", + $"--name=fio_randwrite_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + $"--name=fio_randwrite_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based" + }; - // Setup the expectations for the workload - // - Disks are formatted and ready - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. - this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.Setup(platform); this.mockFixture.SetupDisks(withUnformatted: false); - this.mockFixture.SetupPackage("fio", expectedFiles: $@"win-x64\fio.exe"); + string expectedFiles = platform == PlatformID.Win32NT ? $@"win-x64\fio.exe" : $@"linux-x64/fio"; + this.mockFixture.SetupPackage("fio", expectedFiles: expectedFiles); this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { @@ -112,25 +116,28 @@ public async Task FioWorkloadProfileExecutesTheExpectedWorkloadsOnWindowsPlatfor using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); } } [Test] - [TestCase("PERF-IO-FIO.json")] - public async Task FioWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile) + [TestCase("PERF-IO-FIO-SEQWRITE.json", PlatformID.Win32NT, TestName = "FioSequentialWriteProfileExecutesOnWindows")] + [TestCase("PERF-IO-FIO-SEQWRITE.json", PlatformID.Unix, TestName = "FioSequentialWriteProfileExecutesOnUnix")] + public async Task FioSequentialWriteWorkloadProfileExecutes(string profile, PlatformID platform) { - IEnumerable expectedCommands = FioProfileTests.GetFioProfileExpectedCommands(PlatformID.Unix); + string expectedIoEngine = platform == PlatformID.Win32NT ? "windowsaio" : "libaio"; + IEnumerable expectedCommands = new List + { + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread ", + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread", + $"--name=fio_write_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + $"--name=fio_write_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based" + }; - // Setup the expectations for the workload - // - Disks are formatted and ready - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(platform); this.mockFixture.SetupDisks(withUnformatted: false); - this.mockFixture.SetupPackage("fio", expectedFiles: $@"linux-x64/fio"); + string expectedFiles = platform == PlatformID.Win32NT ? $@"win-x64\fio.exe" : $@"linux-x64/fio"; + this.mockFixture.SetupPackage("fio", expectedFiles: expectedFiles); this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { @@ -146,57 +153,119 @@ public async Task FioWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(s using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); - WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); } } - private static IEnumerable GetFioProfileExpectedCommands(PlatformID platform) + [Test] + [TestCase("PERF-IO-FIO-RANDREAD.json", PlatformID.Win32NT, TestName = "FioRandomReadProfileExecutesOnWindows")] + [TestCase("PERF-IO-FIO-RANDREAD.json", PlatformID.Unix, TestName = "FioRandomReadProfileExecutesOnUnix")] + public async Task FioRandomReadWorkloadProfileExecutes(string profile, PlatformID platform) { string expectedIoEngine = platform == PlatformID.Win32NT ? "windowsaio" : "libaio"; - return new List + IEnumerable expectedCommands = new List { - // Given the test setup created 2 remote disks, we will perform a disk fill on both individually $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread ", $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread", + $"--name=fio_randread_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + $"--name=fio_randread_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based" + }; - // After the disk fill, we execute the FIO commands. - // Random Write tests - $"--name=fio_randwrite_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randwrite_496G_8k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=8k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randwrite_496G_12k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=12k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randwrite_496G_16k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=16k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randwrite_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randwrite --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + this.mockFixture.Setup(platform); + this.mockFixture.SetupDisks(withUnformatted: false); + string expectedFiles = platform == PlatformID.Win32NT ? $@"win-x64\fio.exe" : $@"linux-x64/fio"; + this.mockFixture.SetupPackage("fio", expectedFiles: expectedFiles); - // Sequential Write tests - $"--name=fio_write_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_write_496G_8k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=8k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_write_496G_12k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=12k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_write_496G_16k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=16k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_write_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=write --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("--name=fio", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_FIO.json")); + } - // Random Read tests - $"--name=fio_randread_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randread_496G_8k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=8k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randread_496G_12k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=12k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randread_496G_16k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=16k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_randread_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=randread --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + return process; + }; - // Sequential Read tests + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-FIO-SEQREAD.json", PlatformID.Win32NT, TestName = "FioSequentialReadProfileExecutesOnWindows")] + [TestCase("PERF-IO-FIO-SEQREAD.json", PlatformID.Unix, TestName = "FioSequentialReadProfileExecutesOnUnix")] + public async Task FioSequentialReadWorkloadProfileExecutes(string profile, PlatformID platform) + { + string expectedIoEngine = platform == PlatformID.Win32NT ? "windowsaio" : "libaio"; + IEnumerable expectedCommands = new List + { + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread ", + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread", $"--name=fio_read_496G_4k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=4k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_read_496G_8k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=8k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_read_496G_12k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=12k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_read_496G_16k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=16k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", - $"--name=fio_read_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based", + $"--name=fio_read_496G_1024k_d[0-9]+_th[0-9]+ --size=496G --numjobs=[0-9]+ --rw=read --bs=1024k --iodepth=[0-9]+ --ioengine={expectedIoEngine} --direct=1 --ramp_time=30 --runtime=300 --time_based" + }; + + this.mockFixture.Setup(platform); + this.mockFixture.SetupDisks(withUnformatted: false); + string expectedFiles = platform == PlatformID.Win32NT ? $@"win-x64\fio.exe" : $@"linux-x64/fio"; + this.mockFixture.SetupPackage("fio", expectedFiles: expectedFiles); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("--name=fio", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_FIO.json")); + } - // Disk Integrity Verification tests (random writes + sequential writes) + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } + } + + [Test] + [TestCase("PERF-IO-FIO-INTEGRITY.json", PlatformID.Win32NT, TestName = "FioDataIntegrityProfileExecutesOnWindows")] + [TestCase("PERF-IO-FIO-INTEGRITY.json", PlatformID.Unix, TestName = "FioDataIntegrityProfileExecutesOnUnix")] + public async Task FioDataIntegrityWorkloadProfileExecutes(string profile, PlatformID platform) + { + string expectedIoEngine = platform == PlatformID.Win32NT ? "windowsaio" : "libaio"; + IEnumerable expectedCommands = new List + { + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread ", + $"--name=disk_fill --size=500G --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --thread", $"--name=fio_randwrite_4G_4k_d1_th1_verify --size=4G --numjobs=1 --rw=randwrite --bs=4k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1", - $"--name=fio_randwrite_4G_16k_d1_th1_verify --size=4G --numjobs=1 --rw=randwrite --bs=16k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1", - $"--name=fio_randwrite_4G_1024k_d1_th1_verify --size=4G --numjobs=1 --rw=randwrite --bs=1024k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1", - $"--name=fio_write_4G_4k_d1_th1_verify --size=4G --numjobs=1 --rw=write --bs=4k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1", - $"--name=fio_write_4G_16k_d1_th1_verify --size=4G --numjobs=1 --rw=write --bs=16k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1", - $"--name=fio_write_4G_1024k_d1_th1_verify --size=4G --numjobs=1 --rw=write --bs=1024k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1" + $"--name=fio_write_4G_4k_d1_th1_verify --size=4G --numjobs=1 --rw=write --bs=4k --iodepth=1 --ioengine={expectedIoEngine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1" }; + + this.mockFixture.Setup(platform); + this.mockFixture.SetupDisks(withUnformatted: false); + string expectedFiles = platform == PlatformID.Win32NT ? $@"win-x64\fio.exe" : $@"linux-x64/fio"; + this.mockFixture.SetupPackage("fio", expectedFiles: expectedFiles); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + IProcessProxy process = this.mockFixture.CreateProcess(command, arguments, workingDir); + if (arguments.Contains("--name=fio", StringComparison.OrdinalIgnoreCase)) + { + process.StandardOutput.Append(TestDependencies.GetResourceFileContents("Results_FIO.json")); + } + + return process; + }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None).ConfigureAwait(false); + WorkloadAssert.CommandsExecuted(this.mockFixture, expectedCommands.ToArray()); + } } } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/MLPerfProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/MLPerfProfileTests.cs index 04900bd3e1..f796f140ba 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/MLPerfProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/MLPerfProfileTests.cs @@ -39,7 +39,8 @@ public void MLPerfWorkloadProfileParametersAreInlinedCorrectly(string profile) } [Test] - [TestCase("PERF-GPU-MLPERF.json")] + [TestCase("PERF-GPU-MLPERF-BERT.json", TestName = "MLPerfBertDependenciesAndReboot")] + [TestCase("PERF-GPU-MLPERF-3DUNET.json", TestName = "MLPerf3DUnetDependenciesAndReboot")] public async Task MLPerfWorkloadProfileExecutesTheExpectedDependenciesAndReboot(string profile) { List expectedCommands = new List @@ -73,10 +74,11 @@ public async Task MLPerfWorkloadProfileExecutesTheExpectedDependenciesAndReboot( } [Test] - [TestCase("PERF-GPU-MLPERF.json")] - public async Task MLPerfWorkloadProfileExecutesTheExpectedRemainingDependenciesAfterRebootAndExecuteWorkload(string profile) + [TestCase("PERF-GPU-MLPERF-BERT.json", "bert", TestName = "MLPerfBertExecutesAfterReboot")] + [TestCase("PERF-GPU-MLPERF-3DUNET.json", "3d-unet", TestName = "MLPerf3DUnetExecutesAfterReboot")] + public async Task MLPerfWorkloadProfileExecutesTheExpectedRemainingDependenciesAfterRebootAndExecuteWorkload(string profile, string models) { - IEnumerable expectedCommands = this.GetExpectedCommands(); + IEnumerable expectedCommands = this.GetExpectedCommands(models); this.mockFixture.Setup(PlatformID.Unix); this.mockFixture.SetupDisks(withRemoteDisks: true); @@ -104,7 +106,7 @@ await this.mockFixture.StateManager.SaveStateAsync(expectedStateId, JObject.Pars } } - private IEnumerable GetExpectedCommands() + private IEnumerable GetExpectedCommands(string models = "bert,3d-unet") { string setupCommand = "curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey " + "| sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \\\n " @@ -112,7 +114,7 @@ private IEnumerable GetExpectedCommands() + " sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \\\n " + " sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list"; - return new List + var allCommands = new List { $"sudo bash -c \"{setupCommand}\"", $"sudo apt-get update", @@ -120,19 +122,54 @@ private IEnumerable GetExpectedCommands() $"sudo systemctl restart docker", "sudo usermod -aG docker [a-z]+", @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make clean""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make link_dirs""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_data BENCHMARKS=bert""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_model BENCHMARKS=bert""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make preprocess_data BENCHMARKS=bert""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_data BENCHMARKS=3d-unet""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_model BENCHMARKS=3d-unet""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make preprocess_data BENCHMARKS=3d-unet""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make build""", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=bert --scenarios=Offline,Server,SingleStream --config_ver=default --test_mode=PerformanceOnly --fast'", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=bert --scenarios=Offline,Server,SingleStream --config_ver=default --test_mode=AccuracyOnly --fast'", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=3d-unet --scenarios=Offline,SingleStream --config_ver=default --test_mode=PerformanceOnly --fast'", - @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=3d-unet --scenarios=Offline,SingleStream --config_ver=default --test_mode=AccuracyOnly --fast'" + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make link_dirs""" }; + + var modelsList = models.Split(','); + bool includeBert = modelsList.Contains("bert"); + bool include3DUnet = modelsList.Contains("3d-unet"); + + if (includeBert) + { + allCommands.AddRange(new[] + { + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_data BENCHMARKS=bert""", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_model BENCHMARKS=bert""", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make preprocess_data BENCHMARKS=bert""" + }); + } + + if (include3DUnet) + { + allCommands.AddRange(new[] + { + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_data BENCHMARKS=3d-unet""", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make download_model BENCHMARKS=3d-unet""", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make preprocess_data BENCHMARKS=3d-unet""" + }); + } + + allCommands.Add(@"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make build"""); + + if (includeBert) + { + allCommands.AddRange(new[] + { + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=bert --scenarios=Offline,Server,SingleStream --config_ver=default --test_mode=PerformanceOnly --fast'", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=bert --scenarios=Offline,Server,SingleStream --config_ver=default --test_mode=AccuracyOnly --fast'" + }); + } + + if (include3DUnet) + { + allCommands.AddRange(new[] + { + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=3d-unet --scenarios=Offline,SingleStream --config_ver=default --test_mode=PerformanceOnly --fast'", + @"sudo docker exec -u [a-z]+ mlperf-inference-[a-z]+-x86_64 sudo bash -c ""export MLPERF_SCRATCH_PATH=(.*)/scratch && make run RUN_ARGS='--benchmarks=3d-unet --scenarios=Offline,SingleStream --config_ver=default --test_mode=AccuracyOnly --fast'" + }); + } + + return allCommands; } } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-3DUNET.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-3DUNET.json new file mode 100644 index 0000000000..f4f49cd0f1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-3DUNET.json @@ -0,0 +1,75 @@ +{ + "Description": "MLPerf 3D-UNET Model GPU Performance Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "05:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu", + "SupportedRequireCustomSystemSupport": "A100_PCIe_40GBx8", + "SpecialRequirements": "This is an Nvidia GPU-specialized workload. It depends upon the system having an Nvidia GPU card/chip.", + "Purpose": "Focused test for 3D-UNET model inference" + }, + "Parameters": { + "Username": "", + "DiskFilter": "SizeGreaterThan:1000gb", + "LinuxCudaVersion": "12.4", + "LinuxDriverVersion": "550", + "LinuxLocalRunFile": "https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_550.54.14_linux.run", + "RequireCustomSystemSupport": true + }, + "Actions": [ + { + "Type": "MLPerfExecutor", + "Parameters": { + "Scenario": "inferencemodels-3d-unet", + "Model": "3d-unet", + "Username": "$.Parameters.Username", + "DiskFilter": "$.Parameters.DiskFilter" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneMLCommonsRepo", + "RepoUri": "https://github.com/mlcommons/inference_results_v4.1.git", + "PackageName": "mlperf" + } + }, + { + "Type": "NvidiaCudaInstallation", + "Parameters": { + "Scenario": "InstallNvidiaCuda", + "LinuxCudaVersion": "$.Parameters.LinuxCudaVersion", + "LinuxDriverVersion": "$.Parameters.LinuxDriverVersion", + "Username": "$.Parameters.Username", + "LinuxLocalRunFile": "$.Parameters.LinuxLocalRunFile" + } + }, + { + "Type": "DockerInstallation", + "Parameters": { + "Scenario": "InstallDocker" + } + }, + { + "Type": "NvidiaContainerToolkitInstallation", + "Parameters": { + "Scenario": "InstallNvidiaContainerToolkit" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-BERT.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-BERT.json new file mode 100644 index 0000000000..fc5e962dd2 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-GPU-MLPERF-BERT.json @@ -0,0 +1,76 @@ +{ + "Description": "MLPerf BERT Model GPU Performance Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "05:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu", + "SupportedRequireCustomSystemSupport": "A100_PCIe_40GBx8", + "SpecialRequirements": "This is an Nvidia GPU-specialized workload. It depends upon the system having an Nvidia GPU card/chip.", + "Purpose": "Focused test for BERT model inference" + }, + "Parameters": { + "Username": "", + "DiskFilter": "SizeGreaterThan:1000gb", + "LinuxCudaVersion": "12.4", + "LinuxDriverVersion": "550", + "LinuxLocalRunFile": "https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_550.54.14_linux.run", + "RequireCustomSystemSupport": true + }, + "Actions": [ + { + "Type": "MLPerfExecutor", + "Parameters": { + "Scenario": "inferencemodels-bert", + "Model": "bert", + "Username": "$.Parameters.Username", + "DiskFilter": "$.Parameters.DiskFilter", + "RequireCustomSystemSupport": "$.Parameters.RequireCustomSystemSupport" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneMLCommonsRepo", + "RepoUri": "https://github.com/mlcommons/inference_results_v4.1.git", + "PackageName": "mlperf" + } + }, + { + "Type": "NvidiaCudaInstallation", + "Parameters": { + "Scenario": "InstallNvidiaCuda", + "LinuxCudaVersion": "$.Parameters.LinuxCudaVersion", + "LinuxDriverVersion": "$.Parameters.LinuxDriverVersion", + "Username": "$.Parameters.Username", + "LinuxLocalRunFile": "$.Parameters.LinuxLocalRunFile" + } + }, + { + "Type": "DockerInstallation", + "Parameters": { + "Scenario": "InstallDocker" + } + }, + { + "Type": "NvidiaContainerToolkitInstallation", + "Parameters": { + "Scenario": "InstallNvidiaContainerToolkit" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-FUNCTIONAL.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-FUNCTIONAL.json new file mode 100644 index 0000000000..9717983133 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-FUNCTIONAL.json @@ -0,0 +1,137 @@ +{ + "Description": "DiskSpd I/O Functional Test Profile (Simplified for faster test execution)", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:15:00", + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "Purpose": "Simplified profile for functional testing with reduced action count" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "MetricScenario": "diskspd_randwrite_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -r4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randwrite" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomWrite_1024k_BlockSize", + "MetricScenario": "diskspd_randwrite_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b1024K -r4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randwrite" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialWrite_4k_BlockSize", + "MetricScenario": "diskspd_write_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -si4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,write" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomRead_4k_BlockSize", + "MetricScenario": "diskspd_randread_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -r4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randread" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialRead_4k_BlockSize", + "MetricScenario": "diskspd_read_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -si4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,read" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDiskSpdPackage", + "BlobContainer": "packages", + "BlobName": "diskspd.2.0.21.zip", + "PackageName": "diskspd", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDREAD.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDREAD.json new file mode 100644 index 0000000000..74d4327d54 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDREAD.json @@ -0,0 +1,86 @@ +{ + "Description": "DiskSpd Random Read I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "Purpose": "Focused test for random read operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomRead_4k_BlockSize", + "MetricScenario": "diskspd_randread_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -r4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randread" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomRead_1024k_BlockSize", + "MetricScenario": "diskspd_randread_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b1024k -r4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randread" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDiskSpdPackage", + "BlobContainer": "packages", + "BlobName": "diskspd.2.0.21.zip", + "PackageName": "diskspd", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDWRITE.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDWRITE.json new file mode 100644 index 0000000000..535d7df443 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-RANDWRITE.json @@ -0,0 +1,86 @@ +{ + "Description": "DiskSpd Random Write I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "Purpose": "Focused test for random write operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "MetricScenario": "diskspd_randwrite_{FileSize}_4k_d{ThreadCount}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -r4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randwrite" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "RandomWrite_1024k_BlockSize", + "MetricScenario": "diskspd_randwrite_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b1024k -r4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,randwrite" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDiskSpdPackage", + "BlobContainer": "packages", + "BlobName": "diskspd.2.0.21.zip", + "PackageName": "diskspd", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQREAD.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQREAD.json new file mode 100644 index 0000000000..a72136c3fc --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQREAD.json @@ -0,0 +1,86 @@ +{ + "Description": "DiskSpd Sequential Read I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "Purpose": "Focused test for sequential read operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialRead_4k_BlockSize", + "MetricScenario": "diskspd_read_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -si4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,read" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialRead_1024k_BlockSize", + "MetricScenario": "diskspd_read_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b1024k -si4K -t{ThreadCount} -o{QueueDepth} -w0 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,read" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDiskSpdPackage", + "BlobContainer": "packages", + "BlobName": "diskspd.2.0.21.zip", + "PackageName": "diskspd", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQWRITE.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQWRITE.json new file mode 100644 index 0000000000..d567bf53b0 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-DISKSPD-SEQWRITE.json @@ -0,0 +1,86 @@ +{ + "Description": "DiskSpd Sequential Write I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "Purpose": "Focused test for sequential write operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialWrite_4k_BlockSize", + "MetricScenario": "diskspd_write_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b4K -si4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,write" + } + }, + { + "Type": "DiskSpdExecutor", + "Parameters": { + "Scenario": "SequentialWrite_1024k_BlockSize", + "MetricScenario": "diskspd_write_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "diskspd", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "-c{FileSize} -b1024k -si4K -t{ThreadCount} -o{QueueDepth} -w100 -d{Duration.TotalSeconds} -Suw -W15 -D -L -Rtext", + "Duration": "$.Parameters.Duration", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "FileSize": "$.Parameters.FileSize", + "ProcessModel": "SingleProcess", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,DiskSpd,write" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDiskSpdPackage", + "BlobContainer": "packages", + "BlobName": "diskspd.2.0.21.zip", + "PackageName": "diskspd", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-INTEGRITY.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-INTEGRITY.json new file mode 100644 index 0000000000..11b35fd372 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-INTEGRITY.json @@ -0,0 +1,101 @@ +{ + "Description": "FIO Data Integrity Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", + "Purpose": "Focused test for data integrity verification" + }, + "Parameters": { + "DiskFillSize": "500G", + "DataIntegrityFileSize": "4G", + "DiskFilter": "BiggestSize", + "Engine": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"libaio\" : \"windowsaio\")}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=disk_fill --size={DiskFillSize} --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={Engine} --direct=1 --overwrite=1 --thread", + "Engine": "$.Parameters.Engine", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DataIntegrity_RandomWrite_4k_BlockSize", + "MetricScenario": "fio_randwrite_{FileSize}_4k_d1_th1_verify", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_randwrite_{FileSize}_4k_d1_th1_verify --size={FileSize} --numjobs=1 --rw=randwrite --bs=4k --iodepth=1 --ioengine={Engine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1 --thread --group_reporting --output-format=json", + "Engine": "$.Parameters.Engine", + "TestFocus": "DataIntegrity", + "FileSize": "$.Parameters.DataIntegrityFileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": true, + "Tags": "IO,FIO,randwrite,data integrity" + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DataIntegrity_SequentialWrite_4k_BlockSize", + "MetricScenario": "fio_write_{FileSize}_4k_d1_th1_verify", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_write_{FileSize}_4k_d1_th1_verify --size={FileSize} --numjobs=1 --rw=write --bs=4k --iodepth=1 --ioengine={Engine} --direct=1 --overwrite=1 --verify=sha256 --do_verify=1 --thread --group_reporting --output-format=json", + "Engine": "$.Parameters.Engine", + "TestFocus": "DataIntegrity", + "FileSize": "$.Parameters.DataIntegrityFileSize", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": true, + "Tags": "IO,FIO,write,data integrity" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "fio", + "Packages-Dnf": "fio,lshw,parted", + "Packages-Yum": "fio", + "Packages-Zypper": "fio" + } + }, + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallFIOPackage", + "SupportedPlatforms": "win-x64", + "BlobContainer": "packages", + "BlobName": "fio.3.30.0.zip", + "PackageName": "fio", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDREAD.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDREAD.json new file mode 100644 index 0000000000..52d8947c78 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDREAD.json @@ -0,0 +1,108 @@ +{ + "Description": "FIO Random Read I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", + "Purpose": "Focused test for random read operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "Engine": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"libaio\" : \"windowsaio\")}", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=disk_fill --size={DiskFillSize} --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={Engine} --direct=1 --overwrite=1 --thread", + "Engine": "$.Parameters.Engine", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomRead_4k_BlockSize", + "MetricScenario": "fio_randread_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_randread_{FileSize}_4k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=randread --bs=4k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,randread" + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomRead_1024k_BlockSize", + "MetricScenario": "fio_randread_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_randread_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=randread --bs=1024k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,randread" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "fio", + "Packages-Dnf": "fio,lshw,parted", + "Packages-Yum": "fio", + "Packages-Zypper": "fio" + } + }, + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallFIOPackage", + "SupportedPlatforms": "win-x64", + "BlobContainer": "packages", + "BlobName": "fio.3.30.0.zip", + "PackageName": "fio", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDWRITE.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDWRITE.json new file mode 100644 index 0000000000..6198e0bc78 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-RANDWRITE.json @@ -0,0 +1,108 @@ +{ + "Description": "FIO Random Write I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", + "Purpose": "Focused test for random write operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "Engine": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"libaio\" : \"windowsaio\")}", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=disk_fill --size={DiskFillSize} --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={Engine} --direct=1 --overwrite=1 --thread", + "Engine": "$.Parameters.Engine", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "MetricScenario": "fio_randwrite_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_randwrite_{FileSize}_4k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=randwrite --bs=4k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --overwrite=1 --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,randwrite" + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_1024k_BlockSize", + "MetricScenario": "fio_randwrite_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_randwrite_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=randwrite --bs=1024k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --overwrite=1 --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,randwrite" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "fio", + "Packages-Dnf": "fio,lshw,parted", + "Packages-Yum": "fio", + "Packages-Zypper": "fio" + } + }, + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallFIOPackage", + "SupportedPlatforms": "win-x64", + "BlobContainer": "packages", + "BlobName": "fio.3.30.0.zip", + "PackageName": "fio", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQREAD.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQREAD.json new file mode 100644 index 0000000000..6c0a51b223 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQREAD.json @@ -0,0 +1,108 @@ +{ + "Description": "FIO Sequential Read I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", + "Purpose": "Focused test for sequential read operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "Engine": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"libaio\" : \"windowsaio\")}", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=disk_fill --size={DiskFillSize} --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={Engine} --direct=1 --overwrite=1 --thread", + "Engine": "$.Parameters.Engine", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "SequentialRead_4k_BlockSize", + "MetricScenario": "fio_read_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_read_{FileSize}_4k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=read --bs=4k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,read" + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "SequentialRead_1024k_BlockSize", + "MetricScenario": "fio_read_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_read_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=read --bs=1024k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,read" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "fio", + "Packages-Dnf": "fio,lshw,parted", + "Packages-Yum": "fio", + "Packages-Zypper": "fio" + } + }, + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallFIOPackage", + "SupportedPlatforms": "win-x64", + "BlobContainer": "packages", + "BlobName": "fio.3.30.0.zip", + "PackageName": "fio", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQWRITE.json b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQWRITE.json new file mode 100644 index 0000000000..0dc4a8db94 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/Profiles/PERF-IO-FIO-SEQWRITE.json @@ -0,0 +1,108 @@ +{ + "Description": "FIO Sequential Write I/O Test Profile", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", + "Purpose": "Focused test for sequential write operations" + }, + "Parameters": { + "DiskFillSize": "500G", + "FileSize": "496G", + "DiskFilter": "BiggestSize", + "Duration": "00:05:00", + "Engine": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"libaio\" : \"windowsaio\")}", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "QueueDepth": "{calculate(512/{ThreadCount})}", + "ProcessModel": "SingleProcess" + }, + "Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "DiskFill", + "MetricScenario": "disk_fill", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=disk_fill --size={DiskFillSize} --numjobs=1 --rw=write --bs=256k --iodepth=64 --ioengine={Engine} --direct=1 --overwrite=1 --thread", + "Engine": "$.Parameters.Engine", + "DiskFill": true, + "DiskFillSize": "$.Parameters.DiskFillSize", + "ProcessModel": "SingleProcessPerDisk", + "DeleteTestFilesOnFinish": false + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "SequentialWrite_4k_BlockSize", + "MetricScenario": "fio_write_{FileSize}_4k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_write_{FileSize}_4k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=write --bs=4k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --overwrite=1 --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,write" + } + }, + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "SequentialWrite_1024k_BlockSize", + "MetricScenario": "fio_write_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount}", + "PackageName": "fio", + "DiskFilter": "$.Parameters.DiskFilter", + "CommandLine": "--name=fio_write_{FileSize}_1024k_d{QueueDepth}_th{ThreadCount} --size={FileSize} --numjobs={ThreadCount} --rw=write --bs=1024k --iodepth={QueueDepth} --ioengine={Engine} --direct=1 --ramp_time=30 --runtime={Duration.TotalSeconds} --time_based --overwrite=1 --thread --group_reporting --output-format=json", + "ThreadCount": "$.Parameters.ThreadCount", + "QueueDepth": "$.Parameters.QueueDepth", + "Engine": "$.Parameters.Engine", + "FileSize": "$.Parameters.FileSize", + "Duration": "$.Parameters.Duration", + "ProcessModel": "$.Parameters.ProcessModel", + "DeleteTestFilesOnFinish": false, + "Tags": "IO,FIO,write" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "fio", + "Packages-Dnf": "fio,lshw,parted", + "Packages-Yum": "fio", + "Packages-Zypper": "fio" + } + }, + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "InitializeDisks" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "DiskFilter": "$.Parameters.DiskFilter" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallFIOPackage", + "SupportedPlatforms": "win-x64", + "BlobContainer": "packages", + "BlobName": "fio.3.30.0.zip", + "PackageName": "fio", + "Extract": true + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj index c268f5a10b..c19a95c850 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj @@ -30,6 +30,7 @@ +