Add --builders to limit number of projects can build concurrently (#2307)

Author: sheetalkamat

internal/core/buildoptions.go

@@ -6,6 +6,7 @@ type BuildOptions struct {
 	Dry               Tristate `json:"dry,omitzero"`
 	Force             Tristate `json:"force,omitzero"`
 	Verbose           Tristate `json:"verbose,omitzero"`
+	Builders          *int     `json:"builders,omitzero"`
 	StopBuildOnErrors Tristate `json:"stopBuildOnErrors,omitzero"`
 
 	// CompilerOptions are not parsed here and will be available on ParsedBuildCommandLine

internal/diagnostics/diagnostics_generated.go

@@ -35,6 +35,14 @@
         "category": "Message",
         "code": 100008
     },
+    "Set the number of projects to build concurrently.": {
+        "category": "Message",
+        "code": 100009
+    },
+    "all, unless --singleThreaded is passed.": {
+        "category": "Message",
+        "code": 100010
+    },
     "Non-relative paths are not allowed. Did you forget a leading './'?": {
         "category": "Error",
         "code": 5090
@@ -74,5 +82,9 @@
     "Option '--incremental' is only valid with a known configuration file (like 'tsconfig.json') or when '--tsBuildInfoFile' is explicitly provided.": {
         "category": "Error",
         "code": 5074
+    },
+    "Option '{0}' requires value to be greater than '{1}'.": {
+        "category": "Error",
+        "code": 5002
     }
 }

internal/diagnostics/extraDiagnosticMessages.json

@@ -200,6 +200,10 @@ func (t *BuildTask) updateDownstream(orchestrator *Orchestrator, path tspath.Pat
 }
 
 func (t *BuildTask) compileAndEmit(orchestrator *Orchestrator, path tspath.Path) {
+	if orchestrator.buildSemaphore != nil {
+		orchestrator.buildSemaphore <- struct{}{}        // acquire slot
+		defer func() { <-orchestrator.buildSemaphore }() // release slot
+	}
 	t.errors = nil
 	if orchestrator.opts.Command.BuildOptions.Verbose.IsTrue() {
 		t.result.reportStatus(ast.NewCompilerDiagnostic(diagnostics.Building_project_0, orchestrator.relativeFileName(t.config)))

internal/execute/build/buildtask.go

@@ -61,6 +61,8 @@ type Orchestrator struct {
 	tasks  *collections.SyncMap[tspath.Path, *BuildTask]
 	order  []string
 	errors []*ast.Diagnostic
+	// Semaphore to limit concurrent builds
+	buildSemaphore chan struct{}
 
 	errorSummaryReporter tsc.DiagnosticsReporter
 	watchStatusReporter  tsc.DiagnosticReporter
@@ -396,5 +398,9 @@ func NewOrchestrator(opts Options) *Orchestrator {
 	} else {
 		orchestrator.errorSummaryReporter = tsc.CreateReportErrorSummary(opts.Sys, opts.Command.Locale(), opts.Command.CompilerOptions)
 	}
+	// If we want to build more than one project at a time, create a semaphore to limit concurrency
+	if builders := opts.Command.BuildOptions.Builders; builders != nil {
+		orchestrator.buildSemaphore = make(chan struct{}, *builders)
+	}
 	return orchestrator
 }

internal/execute/build/orchestrator.go

@@ -3,6 +3,7 @@ package tsctests
 import (
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -2174,7 +2175,7 @@ func TestBuildProjectsBuilding(t *testing.T) {
 		return files
 	}
 
-	getTestCases := func(pkgCount int) []*tscInput {
+	getTestCases := func(pkgCount int, builders int) []*tscInput {
 		edits := []*tscEdit{
 			{
 				caption: "dts doesn't change",
@@ -2199,21 +2200,35 @@ func TestBuildProjectsBuilding(t *testing.T) {
 				commandLineArgs: []string{"-b", "-v"},
 				edits:           edits,
 			},
+			{
+				subScenario:     fmt.Sprintf(`when there are %d projects in a solution with --builders %d`, pkgCount, builders),
+				files:           files(pkgCount),
+				cwd:             "/user/username/projects/myproject",
+				commandLineArgs: []string{"-b", "-v", "--builders", strconv.Itoa(builders)},
+				edits:           edits,
+			},
 			{
 				subScenario:     fmt.Sprintf(`when there are %d projects in a solution`, pkgCount),
 				files:           files(pkgCount),
 				cwd:             "/user/username/projects/myproject",
 				commandLineArgs: []string{"-b", "-w", "-v"},
 				edits:           edits,
 			},
+			{
+				subScenario:     fmt.Sprintf(`when there are %d projects in a solution with --builders %d`, pkgCount, builders),
+				files:           files(pkgCount),
+				cwd:             "/user/username/projects/myproject",
+				commandLineArgs: []string{"-b", "-w", "-v", "--builders", strconv.Itoa(builders)},
+				edits:           edits,
+			},
 		}
 	}
 
 	testCases := slices.Concat(
-		getTestCases(3),
-		getTestCases(5),
-		getTestCases(8),
-		getTestCases(23),
+		getTestCases(3, 1),
+		getTestCases(5, 2),
+		getTestCases(8, 3),
+		getTestCases(23, 3),
 	)
 
 	for _, test := range testCases {

internal/execute/tsctests/tscbuild_test.go

@@ -38,6 +38,9 @@ type CommandLineOption struct {
 	// What kind of extra validation `validateJsonOptionValue` should do
 	extraValidation extraValidation
 
+	// checks that option with number type has value >= minValue
+	minValue int
+
 	// true or undefined
 	// used for configDirTemplateSubstitutionOptions
 	allowConfigDirTemplateSubstitution bool

internal/tsoptions/commandlineoption.go

@@ -142,7 +142,7 @@ func (p *commandLineParser) parseStrings(args []string) {
 			inputOptionName := getInputOptionName(s)
 			opt := p.optionsMap.GetOptionDeclarationFromName(inputOptionName, true /*allowShort*/)
 			if opt != nil {
-				i = p.parseOptionValue(args, i, opt, nil)
+				i = p.parseOptionValue(args, i, opt, p.workerDiagnostics.OptionTypeMismatchDiagnostic)
 			} else {
 				watchOpt := WatchNameMap.GetOptionDeclarationFromName(inputOptionName, true /*allowShort*/)
 				if watchOpt != nil {
@@ -256,9 +256,6 @@ func (p *commandLineParser) parseOptionValue(
 		// Check to see if no argument was provided (e.g. "--locale" is the last command-line argument).
 		if i >= len(args) {
 			if opt.Kind != "boolean" {
-				if diag == nil {
-					diag = p.workerDiagnostics.OptionTypeMismatchDiagnostic
-				}
 				p.errors = append(p.errors, ast.NewCompilerDiagnostic(diag, opt.Name, getCompilerOptionValueTypeString(opt)))
 				if opt.Kind == "list" {
 					p.options.Set(opt.Name, []string{})
@@ -276,7 +273,13 @@ func (p *commandLineParser) parseOptionValue(
 				// !!! Make sure this parseInt matches JS parseInt
 				num, e := strconv.Atoi(args[i])
 				if e == nil {
-					p.options.Set(opt.Name, num)
+					if num >= opt.minValue {
+						p.options.Set(opt.Name, num)
+					} else {
+						p.errors = append(p.errors, ast.NewCompilerDiagnostic(diagnostics.Option_0_requires_value_to_be_greater_than_1, opt.Name, strconv.Itoa(opt.minValue)))
+					}
+				} else {
+					p.errors = append(p.errors, ast.NewCompilerDiagnostic(diag, opt.Name, "number"))
 				}
 				i++
 			case "boolean":

internal/tsoptions/commandlineparser.go

@@ -292,11 +292,22 @@ func formatNewBaseline(
 }
 
 func (f commandLineSubScenario) assertBuildParseResult(t *testing.T) {
+	t.Helper()
+	f.assertBuildParseResultWithTsBaseline(t, func() *TestCommandLineParserBuild {
+		originalBaseline := f.baseline.ReadFile(t)
+		return parseExistingCompilerBaselineBuild(t, originalBaseline)
+	})
+}
+
+func (f commandLineSubScenario) assertBuildParseResultWithTsBaseline(t *testing.T, getTsBaseline func() *TestCommandLineParserBuild) {
 	t.Helper()
 	t.Run(f.testName, func(t *testing.T) {
 		t.Parallel()
-		originalBaseline := f.baseline.ReadFile(t)
-		tsBaseline := parseExistingCompilerBaselineBuild(t, originalBaseline)
+
+		var tsBaseline *TestCommandLineParserBuild
+		if getTsBaseline != nil {
+			tsBaseline = getTsBaseline()
+		}
 
 		// f.workerDiagnostic is either defined or set to default pointer in `createSubScenario`
 		parsed := tsoptions.ParseBuildCommandLine(f.commandLine, &tsoptionstest.VfsParseConfigHost{
@@ -305,19 +316,25 @@ func (f commandLineSubScenario) assertBuildParseResult(t *testing.T) {
 		})
 
 		newBaselineProjects := strings.Join(parsed.Projects, ",")
-		assert.Equal(t, tsBaseline.projects, newBaselineProjects)
+		if getTsBaseline != nil {
+			assert.Equal(t, tsBaseline.projects, newBaselineProjects)
+		}
 
 		o, _ := json.Marshal(parsed.BuildOptions)
 		newParsedBuildOptions := &core.BuildOptions{}
 		e := json.Unmarshal(o, newParsedBuildOptions)
 		assert.NilError(t, e)
-		assert.DeepEqual(t, tsBaseline.options, newParsedBuildOptions, cmpopts.IgnoreUnexported(core.BuildOptions{}))
+		if getTsBaseline != nil {
+			assert.DeepEqual(t, tsBaseline.options, newParsedBuildOptions, cmpopts.IgnoreUnexported(core.BuildOptions{}))
+		}
 
 		compilerOpts, _ := json.Marshal(parsed.CompilerOptions)
 		newParsedCompilerOptions := &core.CompilerOptions{}
 		e = json.Unmarshal(compilerOpts, newParsedCompilerOptions)
 		assert.NilError(t, e)
-		assert.DeepEqual(t, tsBaseline.compilerOptions, newParsedCompilerOptions, cmpopts.IgnoreUnexported(core.CompilerOptions{}))
+		if getTsBaseline != nil {
+			assert.DeepEqual(t, tsBaseline.compilerOptions, newParsedCompilerOptions, cmpopts.IgnoreUnexported(core.CompilerOptions{}))
+		}
 
 		newParsedWatchOptions := core.WatchOptions{}
 		e = json.Unmarshal(o, &newParsedWatchOptions)
@@ -478,6 +495,18 @@ func TestParseBuildCommandLine(t *testing.T) {
 	for _, testCase := range parseCommandLineSubScenarios {
 		testCase.createSubScenario("parseBuildOptions").assertBuildParseResult(t)
 	}
+
+	extraScenarios := []*subScenarioInput{
+		{`parse --builders`, []string{"--builders", "2"}},
+		{`--singleThreaded and --builders together`, []string{"--singleThreaded", "--builders", "2"}},
+		{`reports error when --builders is 0`, []string{"--builders", "0"}},
+		{`reports error when --builders is negative`, []string{"--builders", "-1"}},
+		{`reports error when --builders is invalid type`, []string{"--builders", "invalid"}},
+	}
+
+	for _, testCase := range extraScenarios {
+		testCase.createSubScenario("parseBuildOptions").assertBuildParseResultWithTsBaseline(t, nil)
+	}
 }
 
 func TestAffectsBuildInfo(t *testing.T) {

internal/tsoptions/commandlineparser_test.go

@@ -49,6 +49,14 @@ var OptionsForBuild = []*CommandLineOption{
 		Kind:                    "boolean",
 		DefaultValueDescription: false,
 	},
+	{
+		Name:                    "builders",
+		Kind:                    CommandLineOptionTypeNumber,
+		Category:                diagnostics.Command_line_Options,
+		Description:             diagnostics.Set_the_number_of_projects_to_build_concurrently,
+		DefaultValueDescription: diagnostics.X_all_unless_singleThreaded_is_passed,
+		minValue:                1,
+	},
 	{
 		Name:                    "stopBuildOnErrors",
 		Category:                diagnostics.Command_line_Options,