Add --repo-list to generate-script (#468)

RELEASENOTES.md

@@ -1,3 +1,4 @@
+- added `--repo-list` argument to `ado2gh generate-script`. This accepts a repos.csv file previously generated by `ado2gh inventory-report`. This can be used to split a large migration up into batches of repos.
 - `--github-pat` arg in the `gei reclaim-mannequin` command was renamed to `--github-target-pat` to follow the same naming convention for other commands like ` gei migrate-repo` or `gei generate-mannequin-csv` 
 - `ado2gh inventory-report` command now also reports the compressed size of each repo in `repos.csv`.
 - fixed bug in `gh gei generate-script` so that it properly respects the `--no-ssl-verify` argument.

src/OctoshiftCLI.IntegrationTests/.editorconfig

@@ -0,0 +1,22 @@
+[*.cs]
+
+# CA1707: Identifiers should not contain underscores
+dotnet_diagnostic.CA1707.severity = none
+
+# Naming rules
+
+dotnet_naming_rule.public_methods_should_be_snake_case.severity = suggestion
+dotnet_naming_rule.public_methods_should_be_snake_case.symbols = public_methods
+dotnet_naming_rule.public_methods_should_be_snake_case.style = snake_case
+
+# Symbol specifications
+
+dotnet_naming_symbols.public_methods.applicable_kinds =  method
+dotnet_naming_symbols.public_methods.applicable_accessibilities = public
+
+# Naming styles
+
+dotnet_naming_style.snake_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =

src/OctoshiftCLI.IntegrationTests/AdoToGithub.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Net.Http;
 using System.Threading.Tasks;
 using Xunit;
@@ -48,6 +49,66 @@ public AdoToGithub(ITestOutputHelper output)
             _helper = new TestHelper(_output, adoApi, githubApi, adoClient, githubClient);
         }
 
+        [Fact]
+        public async Task With_Inventory_Report_Csv()
+        {
+            var adoOrg = $"gei-e2e-testing-{TestHelper.GetOsName()}";
+            var githubOrg = $"e2e-testing-{TestHelper.GetOsName()}";
+            var teamProject1 = "gei-e2e-1";
+            var teamProject2 = "gei-e2e-2";
+            var adoRepo1 = teamProject1;
+            var adoRepo2 = teamProject2;
+            var pipeline1 = "pipeline1";
+            var pipeline2 = "pipeline2";
+
+            await _helper.ResetAdoTestEnvironment(adoOrg);
+            await _helper.ResetGithubTestEnvironment(githubOrg);
+
+            await _helper.CreateTeamProject(adoOrg, teamProject1);
+            var commitId = await _helper.InitializeAdoRepo(adoOrg, teamProject1, adoRepo1);
+            await _helper.CreatePipeline(adoOrg, teamProject1, adoRepo1, pipeline1, commitId);
+
+            await _helper.CreateTeamProject(adoOrg, teamProject2);
+            commitId = await _helper.InitializeAdoRepo(adoOrg, teamProject2, adoRepo2);
+            await _helper.CreatePipeline(adoOrg, teamProject2, adoRepo2, pipeline2, commitId);
+
+            await _helper.RunCliCommand($"inventory-report --ado-org {adoOrg}", Path.Join(TestHelper.GetOsDistPath(), "ado2gh"), _tokens);
+            await _helper.RunAdoToGithubCliMigration($"generate-script --github-org {githubOrg} --ado-org {adoOrg} --all --repo-list repos.csv", _tokens);
+
+            _helper.AssertNoErrorInLogs(_startTime);
+
+            await _helper.AssertGithubRepoExists(githubOrg, $"{teamProject1}-{teamProject1}");
+            await _helper.AssertGithubRepoExists(githubOrg, $"{teamProject2}-{teamProject2}");
+            await _helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject1}-{teamProject1}");
+            await _helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject2}-{teamProject2}");
+            await _helper.AssertAutolinkConfigured(githubOrg, $"{teamProject1}-{teamProject1}", $"https://dev.azure.com/{adoOrg}/{teamProject1}/_workitems/edit/<num>/");
+            await _helper.AssertAutolinkConfigured(githubOrg, $"{teamProject2}-{teamProject2}", $"https://dev.azure.com/{adoOrg}/{teamProject2}/_workitems/edit/<num>/");
+            await _helper.AssertAdoRepoDisabled(adoOrg, teamProject1, adoRepo1);
+            await _helper.AssertAdoRepoDisabled(adoOrg, teamProject2, adoRepo2);
+            await _helper.AssertAdoRepoLocked(adoOrg, teamProject1, adoRepo1);
+            await _helper.AssertAdoRepoLocked(adoOrg, teamProject2, adoRepo2);
+            await _helper.AssertGithubTeamCreated(githubOrg, $"{teamProject1}-Maintainers");
+            await _helper.AssertGithubTeamCreated(githubOrg, $"{teamProject1}-Admins");
+            await _helper.AssertGithubTeamCreated(githubOrg, $"{teamProject2}-Maintainers");
+            await _helper.AssertGithubTeamCreated(githubOrg, $"{teamProject2}-Admins");
+            await _helper.AssertGithubTeamIdpLinked(githubOrg, $"{teamProject1}-Maintainers", $"{teamProject1}-Maintainers");
+            await _helper.AssertGithubTeamIdpLinked(githubOrg, $"{teamProject1}-Admins", $"{teamProject1}-Admins");
+            await _helper.AssertGithubTeamIdpLinked(githubOrg, $"{teamProject2}-Maintainers", $"{teamProject2}-Maintainers");
+            await _helper.AssertGithubTeamIdpLinked(githubOrg, $"{teamProject2}-Admins", $"{teamProject2}-Admins");
+            await _helper.AssertGithubTeamHasRepoRole(githubOrg, $"{teamProject1}-Maintainers", $"{teamProject1}-{teamProject1}", "maintain");
+            await _helper.AssertGithubTeamHasRepoRole(githubOrg, $"{teamProject1}-Admins", $"{teamProject1}-{teamProject1}", "admin");
+            await _helper.AssertGithubTeamHasRepoRole(githubOrg, $"{teamProject2}-Maintainers", $"{teamProject2}-{teamProject2}", "maintain");
+            await _helper.AssertGithubTeamHasRepoRole(githubOrg, $"{teamProject2}-Admins", $"{teamProject2}-{teamProject2}", "admin");
+            await _helper.AssertServiceConnectionWasShared(adoOrg, teamProject1);
+            await _helper.AssertServiceConnectionWasShared(adoOrg, teamProject2);
+            await _helper.AssertPipelineRewired(adoOrg, teamProject1, pipeline1, githubOrg, $"{teamProject1}-{teamProject1}");
+            await _helper.AssertPipelineRewired(adoOrg, teamProject2, pipeline2, githubOrg, $"{teamProject2}-{teamProject2}");
+            await _helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject1);
+            await _helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject2);
+            _helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject1}-{teamProject1}");
+            _helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject2}-{teamProject2}");
+        }
+
         [Fact]
         public async Task Basic()
         {

src/OctoshiftCLI.IntegrationTests/TestHelper.cs

@@ -426,11 +426,19 @@ public static string GetOsName()
 
         public async Task RunCliMigration(string generateScriptCommand, string cliName, IDictionary<string, string> tokens)
         {
+            await RunCliCommand(generateScriptCommand, cliName, tokens);
+            await RunPowershellScript("migrate.ps1", tokens);
+        }
+
+        public async Task RunPowershellScript(string script, IDictionary<string, string> tokens)
+        {
+            var scriptPath = Path.Join(GetOsDistPath(), script);
+
             var startInfo = new ProcessStartInfo
             {
                 WorkingDirectory = GetOsDistPath(),
-                FileName = $"{cliName}",
-                Arguments = generateScriptCommand
+                FileName = "pwsh",
+                Arguments = $"-File {scriptPath}"
             };
 
             if (tokens != null)
@@ -449,20 +457,42 @@ public async Task RunCliMigration(string generateScriptCommand, string cliName,
             }
 
             _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}");
+
             var p = Process.Start(startInfo);
             await p.WaitForExitAsync();
 
-            p.ExitCode.Should().Be(0, "generate-script should return an exit code of 0");
+            p.ExitCode.Should().Be(0, $"{script} should return an exit code of 0");
+        }
 
-            startInfo.FileName = "pwsh";
-            var scriptPath = Path.Join(startInfo.WorkingDirectory, "migrate.ps1");
-            startInfo.Arguments = $"-File {scriptPath}";
+        public async Task RunCliCommand(string command, string cliName, IDictionary<string, string> tokens)
+        {
+            var startInfo = new ProcessStartInfo
+            {
+                WorkingDirectory = GetOsDistPath(),
+                FileName = cliName,
+                Arguments = command
+            };
+
+            if (tokens != null)
+            {
+                foreach (var token in tokens)
+                {
+                    if (startInfo.EnvironmentVariables.ContainsKey(token.Key))
+                    {
+                        startInfo.EnvironmentVariables[token.Key] = token.Value;
+                    }
+                    else
+                    {
+                        startInfo.EnvironmentVariables.Add(token.Key, token.Value);
+                    }
+                }
+            }
 
             _output.WriteLine($"Running command: {startInfo.FileName} {startInfo.Arguments}");
-            p = Process.Start(startInfo);
+            var p = Process.Start(startInfo);
             await p.WaitForExitAsync();
 
-            p.ExitCode.Should().Be(0, "migrate.ps1 should return an exit code of 0");
+            p.ExitCode.Should().Be(0, $"{cliName} should return an exit code of 0");
         }
 
         public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IDictionary<string, string> tokens) =>
@@ -471,7 +501,7 @@ public async Task RunAdoToGithubCliMigration(string generateScriptCommand, IDict
         public async Task RunGeiCliMigration(string generateScriptCommand, IDictionary<string, string> tokens) =>
             await RunCliMigration($"gei {generateScriptCommand}", "gh", tokens);
 
-        private static string GetOsDistPath()
+        public static string GetOsDistPath()
         {
             return RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
                 ? Path.Join(Directory.GetCurrentDirectory(), "../../../../../dist/linux-x64")

src/OctoshiftCLI.Tests/TestExtensionMethods.cs

@@ -1,5 +1,7 @@
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text;
 using Newtonsoft.Json.Linq;
 
 namespace OctoshiftCLI.Tests
@@ -8,5 +10,10 @@ public static class TestExtensionMethods
     {
         public static IAsyncEnumerable<JToken> ToAsyncJTokenEnumerable<T>(this IEnumerable<T> list)
             => list.Select(x => JToken.FromObject(x)).ToAsyncEnumerable();
+
+        public static MemoryStream ToStream(this string value)
+        {
+            return new MemoryStream(Encoding.UTF8.GetBytes(value ?? ""));
+        }
     }
 }

src/OctoshiftCLI.Tests/ado2gh/Commands/GenerateScriptCommandTests.cs

@@ -62,7 +62,7 @@ public void Should_Have_Options()
             var command = new GenerateScriptCommand(null, null, null, null);
             command.Should().NotBeNull();
             command.Name.Should().Be("generate-script");
-            command.Options.Count.Should().Be(16);
+            command.Options.Count.Should().Be(17);
 
             TestHelpers.VerifyCommandOption(command.Options, "github-org", true);
             TestHelpers.VerifyCommandOption(command.Options, "ado-org", false);
@@ -80,6 +80,7 @@ public void Should_Have_Options()
             TestHelpers.VerifyCommandOption(command.Options, "integrate-boards", false);
             TestHelpers.VerifyCommandOption(command.Options, "rewire-pipelines", false);
             TestHelpers.VerifyCommandOption(command.Options, "all", false);
+            TestHelpers.VerifyCommandOption(command.Options, "repo-list", false);
         }
 
         [Fact]
@@ -131,6 +132,38 @@ public async Task SequentialScript_Single_Repo_No_Options()
             _scriptOutput.Should().Be(expected);
         }
 
+        [Fact]
+        public async Task SequentialScript_With_RepoList()
+        {
+            // Arrange
+            var repoList = new FileInfo("repos.csv");
+
+            _mockAdoApiFactory.Setup(m => m.Create(null)).Returns(_mockAdoApi.Object);
+
+            _mockAdoInspector.Setup(m => m.GetRepoCount()).ReturnsAsync(1);
+            _mockAdoInspector.Setup(m => m.GetOrgs()).ReturnsAsync(ADO_ORGS);
+            _mockAdoInspector.Setup(m => m.GetTeamProjects(ADO_ORG)).ReturnsAsync(ADO_TEAM_PROJECTS);
+            _mockAdoInspector.Setup(m => m.GetRepos(ADO_ORG, ADO_TEAM_PROJECT)).ReturnsAsync(ADO_REPOS);
+
+            // Act
+            var args = new GenerateScriptCommandArgs
+            {
+                GithubOrg = GITHUB_ORG,
+                AdoOrg = ADO_ORG,
+                Sequential = true,
+                Output = new FileInfo("unit-test-output"),
+                RepoList = repoList
+            };
+            await _command.Invoke(args);
+
+            _scriptOutput = TrimNonExecutableLines(_scriptOutput);
+            var expected = $"Exec {{ ./ado2gh migrate-repo --ado-org \"{ADO_ORG}\" --ado-team-project \"{ADO_TEAM_PROJECT}\" --ado-repo \"{FOO_REPO}\" --github-org \"{GITHUB_ORG}\" --github-repo \"{ADO_TEAM_PROJECT}-{FOO_REPO}\" --wait }}";
+
+            // Assert
+            _scriptOutput.Should().Be(expected);
+            _mockAdoInspector.Verify(m => m.LoadReposCsv(repoList.FullName));
+        }
+
         [Fact]
         public async Task SequentialScript_Single_Repo_All_Options()
         {

src/OctoshiftCLI.Tests/ado2gh/Services/AdoInspectorServiceTests.cs

@@ -120,5 +120,53 @@ public async Task GetPipelines_Should_Return_All_Pipelines()
             // Assert
             result.Should().BeEquivalentTo(pipelines);
         }
+
+        [Fact]
+        public async Task LoadReposCsv_Should_Set_Orgs()
+        {
+            // Arrange
+            var csvPath = "repos.csv";
+            var csvContents = $"org,teamproject,repo{Environment.NewLine}\"{ADO_ORG}\",\"{ADO_TEAM_PROJECT}\",\"{FOO_REPO}\"";
+
+            _service.OpenFileStream = _ => csvContents.ToStream();
+
+            // Act
+            _service.LoadReposCsv(csvPath);
+
+            // Assert
+            (await _service.GetOrgs()).Should().BeEquivalentTo(new List<string>() { ADO_ORG });
+        }
+
+        [Fact]
+        public async Task LoadReposCsv_Should_Set_TeamProjects()
+        {
+            // Arrange
+            var csvPath = "repos.csv";
+            var csvContents = $"org,teamproject,repo{Environment.NewLine}\"{ADO_ORG}\",\"{ADO_TEAM_PROJECT}\",\"{FOO_REPO}\"";
+
+            _service.OpenFileStream = _ => csvContents.ToStream();
+
+            // Act
+            _service.LoadReposCsv(csvPath);
+
+            // Assert
+            (await _service.GetTeamProjects(ADO_ORG)).Should().BeEquivalentTo(new List<string>() { ADO_TEAM_PROJECT });
+        }
+
+        [Fact]
+        public async Task LoadReposCsv_Should_Set_Repos()
+        {
+            // Arrange
+            var csvPath = "repos.csv";
+            var csvContents = $"org,teamproject,repo{Environment.NewLine}\"{ADO_ORG}\",\"{ADO_TEAM_PROJECT}\",\"{FOO_REPO}\"";
+
+            _service.OpenFileStream = _ => csvContents.ToStream();
+
+            // Act
+            _service.LoadReposCsv(csvPath);
+
+            // Assert
+            (await _service.GetRepos(ADO_ORG, ADO_TEAM_PROJECT)).Single().Name.Should().Be(FOO_REPO);
+        }
     }
 }

src/ado2gh/Commands/GenerateScriptCommand.cs

@@ -108,6 +108,11 @@ public GenerateScriptCommand(OctoLogger log, AdoApiFactory adoApiFactory, IVersi
                 IsRequired = false,
                 Description = "Includes all script generation options."
             };
+            var repoList = new Option<FileInfo>("--repo-list")
+            {
+                IsRequired = false,
+                Description = "Path to a csv file that contains a list of repos to generate a script for. The CSV file should be generated using the inventory-report command."
+            };
 
             AddOption(githubOrgOption);
             AddOption(adoOrgOption);
@@ -125,6 +130,7 @@ public GenerateScriptCommand(OctoLogger log, AdoApiFactory adoApiFactory, IVersi
             AddOption(integrateBoards);
             AddOption(rewirePipelines);
             AddOption(all);
+            AddOption(repoList);
 
             Handler = CommandHandler.Create<GenerateScriptCommandArgs>(Invoke);
         }
@@ -158,6 +164,12 @@ public async Task Invoke(GenerateScriptCommandArgs args)
             _adoInspectorService.OrgFilter = args.AdoOrg;
             _adoInspectorService.TeamProjectFilter = args.AdoTeamProject;
 
+            if (args.RepoList.HasValue())
+            {
+                _log.LogInformation($"Loading Repo CSV File...");
+                _adoInspectorService.LoadReposCsv(args.RepoList.FullName);
+            }
+
             if (await _adoInspectorService.GetRepoCount() == 0)
             {
                 _log.LogError("A migration script could not be generated because no migratable repos were found. Please note that the GEI does not migrate disabled or TFVC repos.");
@@ -564,6 +576,10 @@ private void LogOptions(GenerateScriptCommandArgs args)
             {
                 _log.LogInformation("ALL: true");
             }
+            if (args.RepoList.HasValue())
+            {
+                _log.LogInformation($"REPO LIST: {args.RepoList}");
+            }
         }
 
         private class GenerateScriptOptions
@@ -638,5 +654,6 @@ public class GenerateScriptCommandArgs
         public bool IntegrateBoards { get; set; }
         public bool RewirePipelines { get; set; }
         public bool All { get; set; }
+        public FileInfo RepoList { get; set; }
     }
 }