feat: Adds generators project and AsyncPaginationExtension generator (#2537)

.devcontainer/Dockerfile

@@ -7,6 +7,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/dotnet
 # "install" the dotnet 3.1 & 5.0 runtime for tests
 COPY --from=mcr.microsoft.com/dotnet/sdk:3.1 /usr/share/dotnet/shared /usr/share/dotnet/shared
 COPY --from=mcr.microsoft.com/dotnet/sdk:5.0 /usr/share/dotnet/shared /usr/share/dotnet/shared
+COPY --from=mcr.microsoft.com/dotnet/sdk:6.0 /usr/share/dotnet/shared /usr/share/dotnet/shared
 
 # # Add mkdocs for doc generation
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get -y install --no-install-recommends python3-pip

.github/workflows/netcore.yml

@@ -23,6 +23,10 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 5.0.*
+    - name: Setup .NET 6
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: 6.0.*
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:

.github/workflows/publish.yml

@@ -17,7 +17,10 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 5.0.*
-
+    - name: Setup .NET 6
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: 6.0.*
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:

.vscode/launch.json

@@ -1,6 +1,16 @@
 {
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Run Generator",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/Octokit.Generators/bin/Debug/net6.0/Octokit.Generators",
+      "args": [],
+      "cwd": "${workspaceFolder}/Octokit.Generators",
+      "console": "internalConsole",
+      "stopAtEntry": false
+    },
     {
       "name": "Run unit tests",
       "type": "coreclr",

Octokit.AsyncPaginationExtension/Octokit.AsyncPaginationExtension.csproj

@@ -18,23 +18,23 @@
     <PackageTags>GitHub API Octokit linqpad-samples dotnetcore</PackageTags>
     <Copyright>Copyright GitHub 2017</Copyright>
     <LangVersion>9</LangVersion>
-	  <Nullable>enable</Nullable>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
-	<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
-		<NetStandardImplicitPackageVersion>2.0.0</NetStandardImplicitPackageVersion>
-	</PropertyGroup>
+  <PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
+    <NetStandardImplicitPackageVersion>2.0.0</NetStandardImplicitPackageVersion>
+  </PropertyGroup>
 
-	<ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
-		<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.2" PrivateAssets="All" />
-	</ItemGroup>
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
+    <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.2" PrivateAssets="All" />
+  </ItemGroup>
 
-	<ItemGroup>
-	  <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
-	</ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
+  </ItemGroup>
 
-	<ItemGroup>
-	  <ProjectReference Include="..\Octokit\Octokit.csproj" />
-	</ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Octokit\Octokit.csproj" />
+  </ItemGroup>
 
 </Project>

Octokit.Generators/AsyncPaginationExtensionsGenerator.cs

@@ -0,0 +1,119 @@
+using System.Linq;
+using System.IO;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Data;
+using System.Text;
+
+namespace Octokit.Generators
+{
+
+  /// <summary>
+  /// AsyncPaginationExtensionsGenerator for generating pagination extensions for Octokit.net Clients that return collections.
+  /// </summary>
+  /// <remarks>
+  /// This generator originally appeared in https://github.com/octokit/octokit.net/pull/2516
+  /// The generator solves a small part of a larger effort that is being discussed:
+  ///  https://github.com/octokit/octokit.net/discussions/2499
+  ///  https://github.com/octokit/octokit.net/discussions/2495
+  ///  https://github.com/octokit/octokit.net/issues/2517
+  ///  In the future, we should be able to unify generation for
+  ///   * models (request and response)
+  ///   * clients
+  ///   * routing and related helpers
+  ///   TODO: Convert to use Rosyln source generators
+  /// </remarks>
+  class AsyncPaginationExtensionsGenerator
+  {
+
+  private const string HEADER = (
+@"using System;
+using System.Collections.Generic;
+
+namespace Octokit.AsyncPaginationExtension
+{
+  /// <summary>
+  /// Provides all extensions for pagination.
+  /// </summary>
+  /// <remarks>
+  /// The <code>pageSize</code> parameter at the end of all methods allows for specifying the amount of elements to be fetched per page.
+  /// Only useful to optimize the amount of API calls made.
+  /// </remarks>
+  public static class Extensions
+  {
+    private const int DEFAULT_PAGE_SIZE = 30;
+  ");
+
+  private const string FOOTER = (
+@"
+  }
+}");
+
+    /// <summary>
+    /// GenerateAsync static entry point for generating pagination extensions.
+    /// </summary>
+    /// <remarks>
+    /// This defaults the search path to the root of the project
+    /// This expects to generate the resulting code and put it in Octokit.AsyncPaginationExtension
+    /// This does a wholesale overwrite on ./Octokit.AsyncPaginationExtension/Extensions.cs
+    /// </remarks>
+    public static async Task GenerateAsync(string root = "./")
+    {
+      var sb = new StringBuilder(HEADER);
+      var enumOptions = new EnumerationOptions { RecurseSubdirectories = true };
+      var paginatedCallRegex = new Regex(@".*Task<IReadOnlyList<(?<returnType>\w+)>>\s*(?<name>\w+)(?<template><.*>)?\((?<arg>.*?)(, )?ApiOptions \w*\);");
+
+      foreach (var file in Directory.EnumerateFiles(root, "I*.cs", enumOptions)) {
+          var type = Path.GetFileNameWithoutExtension(file);
+
+          foreach (var line in File.ReadAllLines(file)) {
+              var match = paginatedCallRegex.Match(line);
+
+              if (!match.Success) { continue; }
+              sb.Append(BuildBodyFromTemplate(match, type));
+          }
+      }
+
+      sb.Append(FOOTER);
+
+      await File.WriteAllTextAsync("./Octokit.AsyncPaginationExtension/Extensions.cs", sb.ToString());
+    }
+
+    /// <summary>
+    /// BuildBodyFromTemplate uses the match from the regex search and parses values from the given source
+    /// to use to generate the paging implementations.
+    /// </summary>
+    /// <remarks>
+    /// TODO: This should be reworked to use source templates
+    /// </remarks>
+    private static string BuildBodyFromTemplate(Match match, string type)
+    {
+      var argSplitRegex = new Regex(@" (?![^<]*>)");
+      var returnType = match.Groups["returnType"].Value;
+      var name = match.Groups["name"].Value;
+      var arg = match.Groups["arg"].Value;
+      var template = match.Groups["template"];
+      var templateStr = template.Success ? template.Value : string.Empty;
+      var splitArgs = argSplitRegex.Split(arg).ToArray();
+
+      var lambda = arg.Length == 0
+          ? $"t.{name}{templateStr}"
+          : $"options => t.{name}{templateStr}({string.Join(' ', splitArgs.Where((_, i) => i % 2 == 1))}, options)";
+
+      var docArgs = string.Join(", ", splitArgs.Where((_, i) => i % 2 == 0)).Replace('<', '{').Replace('>', '}');
+      if (docArgs.Length != 0) {
+          docArgs += ", ";
+      }
+
+      if (arg.Length != 0) {
+          arg += ", ";
+      }
+
+      return ($@"
+    /// <inheritdoc cref=""{type}.{name}({docArgs}ApiOptions)""/>
+    public static IPaginatedList<{returnType}> {name}Async{templateStr}(this {type} t, {arg}int pageSize = DEFAULT_PAGE_SIZE)
+        => pageSize > 0 ? new PaginatedList<{returnType}>({lambda}, pageSize) : throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, ""The page size must be positive."");
+    ");
+    }
+  }
+}

Octokit.Generators/Generator.cs

@@ -0,0 +1,29 @@
+using System.Threading.Tasks;
+
+namespace Octokit.Generators
+{
+  /// <summary>
+  /// Provides an entry point for code generation of various types.
+  /// </summary>
+  /// <remarks>
+  /// The backing source for generation will either be the source files in this repo or
+  /// the OpenAPI Descriptions from the GitHub REST API: https://github.com/github/rest-api-description
+  /// </remarks>
+  class Generator
+    {
+
+        static void Main(string[] args)
+        {
+
+          var operation = args.Length != 0 ? args[0] : "AsyncPaginationExtensions";
+
+          if (operation == "AsyncPaginationExtensions")
+          {
+            Task task = Task.Run( () => AsyncPaginationExtensionsGenerator.GenerateAsync());
+            task.Wait();
+          }
+
+          // Put more generation operations here, convert to case when needed.
+        }
+    }
+}

Octokit.Generators/Octokit.Generators.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Description>A set of code generators for Octokit.NET backed by the GitHub REST API Open API descriptions</Description>
+    <AssemblyTitle>Octokit.Generators</AssemblyTitle>
+    <Authors>GitHub</Authors>
+    <Version>0.0.0-dev</Version>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <AssemblyName>Octokit.Generators</AssemblyName>
+    <PackageId>Octokit.Generators</PackageId>
+    <DebugType>embedded</DebugType>
+    <RepositoryUrl>https://github.com/octokit/octokit.net</RepositoryUrl>
+    <PackageProjectUrl>https://github.com/octokit/octokit.net</PackageProjectUrl>
+    <PackageIconUrl>https://f.cloud.github.com/assets/19977/1510987/64af2b26-4a9d-11e3-89fc-96a185171c75.png</PackageIconUrl>
+    <PackageIcon>octokit.png</PackageIcon>
+    <PackageLicenseExpression>MIT</PackageLicenseExpression>
+    <PackageTags>GitHub API Octokit dotnetcore dotnetstandard2.0</PackageTags>
+    <Copyright>Copyright GitHub 2022</Copyright>
+    <LangVersion>9</LangVersion>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+</Project>

Octokit.Generators/README.md

@@ -0,0 +1,58 @@
+# Octokit.net Generators
+
+## About
+
+We've been discussing and thinking about code generators for a [while now](https://github.com/octokit/octokit.net/discussions/2527) and they are part of the [future vision](https://github.com/octokit/octokit.net/discussions/2495) for where we'd like to head with this SDK.
+
+Consider this to be iteration 0, meaning while we want to potentially move to source generators, templates, and using the features that Rosyln offers we need to do some functional experiments and solve our current needs while iterating on the future.  Acknowledging that code generation is a solved problem (menaing there is existing work out there) we should be intentional with the direction we take.
+
+----
+
+## Getting started
+
+From the Octokit .NET root run:
+
+`dotnet run --project Octokit.Generators`
+
+----
+
+## Debugging
+
+There is a launch config defined for this project named `Run Generator`
+
+----
+
+## CI/Actions
+
+Currently no generation is automatically run as a build/release step.  Once the vision solidifies here a bit we'll begin introducing automation so that whatever is generated is always up to date.
+
+----
+
+## Notes and thoughts on code generation for Octokit.net
+
+### Code generation, interpreters, and language interpolation
+
+Hoisted from this [discussion](https://github.com/octokit/octokit.net/discussions/2495)
+
+As you know there are loads of things that we can do here - especially given the power of .NET, reflection, and Rosyln. Just two thoughts on the cautious side of things here:
+
+1. Just because we can generate things, it does not mean we should. As we roll through discovery on all of this we might find out some areas where generation would be trying to put a square peg in a round hole. We need to have the courage to stand down when needed.
+2. Our long-term goal should be language and platform independence. Meaning, we might nail down incredible generative aspects for the .NET SDK but we should always be targeting things like versioned, package distributed, models that the core SDK can reference as well as a generative engine that could potentially generate the models and implementations based on language templates, RFCs, and the like for any reference language - so generating models and SDK methods should be doable for both .NET and Go (for instance) using the same "engine" if possible. It's lofty but I feel that it could be possible.
+
+So what might the future look like for code generation given the above statements? Let's have a look a what might be a really naive/potential roadmap (again we need the community's input and involvement here - which is why I am grateful for the questions):
+
+1. Make sure our current solution addresses the community needs: All SDKs have been synchronized with the current API surface
+2. Standardize implementations - testing, models, API methods, etc... (this is a critical step to get us to generation)
+3. As we go through the above we will learn more about what works and what doesn't. Armed with that knowledge, begin prototyping generative models for each SDK
+4. Solve OpenAPI generation for at least 1 SDK and implement - again with our sights on other languages
+5. Publish (but don't reference yet) SDK models to packaging platforms (in this case nuget)
+6. Work on Generative API implementations - methods etc..
+7. Model generation is unified in the SDK (i.e. we used the published / versioned models in the SDK) and published to package platforms.
+
+
+After this point, things get really interesting with things like:
+
+- Recipe and workflow generation - think of things like plugins for the SDKs to do things like instantiating project boards with teams and referenced repos all in one SDK call. SDKs shouldn't be reflective API implementations, but rather tools that simplify our approach to getting things done.
+- We could start generating SDKs using language specifications - imagine pairing our OpenAPI specification with a language RFC to spit out a baseline SDK that could be extended.
+- Generating language agnostic SDKs - what if we could maximize code reuse by implementing interpreted recipes or code
+- Building workflow interpolation based on user patterns present in GitHub - this comes back to the community and GitHub joining up to make consistent workflows no matter where the users are - in community apps or GitHub proper.

Octokit.sln

@@ -17,6 +17,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{64FD6CD6
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octokit.AsyncPaginationExtension", "Octokit.AsyncPaginationExtension\Octokit.AsyncPaginationExtension.csproj", "{0E8013E0-0CCF-4433-9E01-51AC288824C5}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Octokit.Generators", "Octokit.Generators\Octokit.Generators.csproj", "{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -47,6 +49,10 @@ Global
 		{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE