feat: support v2 sync protocol

CHANGELOG.md

@@ -17,6 +17,9 @@ and this project adheres to
 
 ### Added
 
+- Add support for sync v2 protocol
+  [#4523](https://github.com/OpenFn/lightning/issues/4523)
+
 ### Changed
 
 ### Fixed

lib/lightning/version_control/project_repo_connection.ex

@@ -22,6 +22,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
     field :branch, :string
     field :access_token, :binary
     field :config_path, :string
+    field :use_yaml_config, :boolean, default: false
     field :accept, :boolean, virtual: true
 
     field :sync_direction, Ecto.Enum,
@@ -56,7 +57,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
   end
 
   @required_fields ~w(github_installation_id repo branch project_id)a
-  @other_fields ~w(config_path)a
+  @other_fields ~w(config_path use_yaml_config)a
 
   def changeset(project_repo_connection, attrs) do
     project_repo_connection
@@ -125,6 +126,14 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
 
   def config_path(repo_connection) do
     repo_connection.config_path ||
-      "./openfn-#{repo_connection.project_id}-config.json"
+      if repo_connection.use_yaml_config do
+        openfn_yaml()
+      else
+        "./openfn-#{repo_connection.project_id}-config.json"
+      end
+  end
+
+  def openfn_yaml do
+    "openfn.yaml"
   end
 end

lib/lightning/version_control/version_control.ex

@@ -11,6 +11,7 @@ defmodule Lightning.VersionControl do
   alias Ecto.Multi
   alias Lightning.Accounts.User
   alias Lightning.Extensions.UsageLimiting
+  alias Lightning.Projects.Project
   alias Lightning.Repo
   alias Lightning.VersionControl.Audit
   alias Lightning.VersionControl.Events
@@ -572,8 +573,15 @@ defmodule Lightning.VersionControl do
 
   defp maybe_create_config_blob(tesla_client, repo_connection) do
     if is_nil(repo_connection.config_path) do
+      content =
+        if repo_connection.use_yaml_config do
+          openfn_yaml(repo_connection)
+        else
+          config_json(repo_connection)
+        end
+
       GithubClient.create_blob(tesla_client, repo_connection.repo, %{
-        content: config_json(repo_connection)
+        content: content
       })
     else
       {:ok, nil}
@@ -625,6 +633,33 @@ defmodule Lightning.VersionControl do
     )
   end
 
+  defp openfn_yaml(repo_connection) do
+    project = Repo.get!(Project, repo_connection.project_id)
+
+    inserted_at = DateTime.to_iso8601(project.inserted_at)
+    updated_at = DateTime.to_iso8601(project.updated_at)
+
+    """
+    project:
+      uuid: #{project.id}
+      endpoint: #{LightningWeb.Endpoint.url()}
+      alias: #{project.env}
+      inserted_at: #{inserted_at}
+      updated_at: #{updated_at}
+      id: #{project.name}
+      name: #{project.name}
+    workspace:
+      credentials: credentials.yaml
+      dirs:
+        projects: .projects
+        workflows: workflows
+      formats:
+        openfn: yaml
+        project: yaml
+        workflow: yaml
+    """
+  end
+
   defp pull_yml_target_path do
     ".github/workflows/openfn-pull.yml"
   end

lib/lightning_web/live/project_live/github_sync_component.ex

@@ -683,6 +683,44 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
     """
   end
 
+  attr :form, :map, required: true
+  attr :project_id, :string, required: true
+
+  defp config_format_toggle(assigns) do
+    ~H"""
+    <div class="mt-4">
+      <details class="group">
+        <summary class="cursor-pointer text-sm text-gray-500 hover:text-gray-700 select-none list-none flex items-center gap-1">
+          <.icon
+            name="hero-chevron-right-mini"
+            class="h-4 w-4 transition-transform group-open:rotate-90"
+          /> Advanced: use new YAML config format
+        </summary>
+        <div class="mt-2 ml-5 p-3 bg-gray-50 rounded-md border border-gray-200">
+          <label class="flex items-center gap-3 cursor-pointer">
+            <.input
+              type="checkbox"
+              field={@form[:use_yaml_config]}
+              hidden_input={false}
+            />
+            <span class="text-sm text-gray-700">
+              Use new <code>openfn.yaml</code> format
+            </span>
+          </label>
+          <p class="mt-1 ml-6 text-xs text-gray-500">
+            Only enable this if you want to use the new <code>openfn.yaml</code>
+            format instead of the legacy JSON config.
+          </p>
+        </div>
+      </details>
+    </div>
+    """
+  end
+
+  defp use_yaml_config?(form) do
+    form[:use_yaml_config].value in [true, "true"]
+  end
+
   attr :form, :map, required: true
 
   defp sync_order_radio(assigns) do
@@ -738,9 +776,9 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
                 Import from GitHub (overwrite this project)
               </label>
               <p id="deploy_first_sync_option_description" class="text-gray-500">
-                If you already have <code>config.json</code>
-                and <code>project.yaml</code>
-                files tracked on GitHub and you want to <b>overwrite</b>
+                If you already have an <code>openfn.yaml</code>
+                (or legacy <code>config.json</code>)
+                tracked on GitHub and you want to <b>overwrite</b>
                 this project on OpenFn, you can choose this advanced option.
               </p>
             </div>
@@ -794,9 +832,7 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
             <li>
               <.icon name="hero-document-plus" class="h-4 w-4" />
               <code>
-                ./openfn-{@project.id}-config.json -> {@form[
-                  :branch
-                ].value}
+                {config_filename(@form, @project.id)} -> {@form[:branch].value}
               </code>
             </li>
           <% end %>
@@ -806,4 +842,12 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
     </div>
     """
   end
+
+  defp config_filename(form, project_id) do
+    if use_yaml_config?(form) do
+      ProjectRepoConnection.openfn_yaml()
+    else
+      "openfn-#{project_id}-config.json"
+    end
+  end
 end

lib/lightning_web/live/project_live/github_sync_component.html.heex

@@ -145,17 +145,22 @@
         </.input>
       </div>
       <div class="mt-4">
+        <.sync_order_radio form={f} />
+      </div>
+      <div :if={f[:sync_direction].value == :deploy} class="mt-4">
         <.input
           type="text"
           field={f[:config_path]}
-          label={"Path to config #{if f[:sync_direction].value == :deploy, do: "(required)", else: "(optional)"}"}
+          label="Path to config (required)"
           placeholder={"./openfn-#{@project.id}-config.json"}
           class="placeholder:italic placeholder:text-slate-400"
         />
       </div>
-      <div class="mt-4">
-        <.sync_order_radio form={f} />
-      </div>
+      <.config_format_toggle
+        :if={f[:sync_direction].value != :deploy}
+        form={f}
+        project_id={@project.id}
+      />
       <%= if f[:branch].value do %>
         <.accept_checkbox
           project={@project}

priv/repo/migrations/20260420114808_use_sync_v2.exs

@@ -0,0 +1,9 @@
+defmodule Lightning.Repo.Migrations.UseSyncV2 do
+  use Ecto.Migration
+
+  def change do
+    alter table(:project_repo_connections) do
+      add :use_yaml_config, :boolean, null: false, default: false
+    end
+  end
+end

test/lightning/version_control_test.exs

@@ -617,7 +617,7 @@ defmodule Lightning.VersionControlTest do
                VersionControl.initiate_sync(repo_connection, commit_message)
     end
 
-    test "creates GH workflow dispatch event", %{
+    test "creates GH workflow dispatch event using JSON config (default)", %{
       commit_message: commit_message,
       repo_connection: repo_connection,
       snapshots: [snapshot, other_snapshot]
@@ -644,15 +644,47 @@ defmodule Lightning.VersionControlTest do
       assert :ok = VersionControl.initiate_sync(repo_connection, commit_message)
     end
 
+    test "creates GH workflow dispatch event using YAML config (use_yaml_config: true)",
+         %{
+           commit_message: commit_message,
+           repo_connection: repo_connection,
+           snapshots: [snapshot, other_snapshot]
+         } do
+      yaml_connection =
+        repo_connection
+        |> Ecto.Changeset.change(use_yaml_config: true)
+        |> Lightning.Repo.update!()
+
+      expect_create_installation_token(yaml_connection.github_installation_id)
+      expect_get_repo(yaml_connection.repo)
+
+      expect_create_workflow_dispatch_with_request_body(
+        yaml_connection.repo,
+        "openfn-pull.yml",
+        %{
+          ref: "main",
+          inputs: %{
+            projectId: yaml_connection.project_id,
+            apiSecretName: api_secret_name(yaml_connection),
+            branch: yaml_connection.branch,
+            pathToConfig: path_to_config(yaml_connection),
+            commitMessage: commit_message,
+            snapshots: "#{other_snapshot.id} #{snapshot.id}"
+          }
+        }
+      )
+
+      assert :ok = VersionControl.initiate_sync(yaml_connection, commit_message)
+    end
+
     defp api_secret_name(%{project_id: project_id}) do
       project_id
       |> String.replace("-", "_")
       |> then(&"OPENFN_#{&1}_API_KEY")
     end
 
     defp path_to_config(repo_connection) do
-      repo_connection
-      |> ProjectRepoConnection.config_path()
+      ProjectRepoConnection.config_path(repo_connection)
       |> Path.relative_to(".")
     end
   end

test/lightning_web/live/sandbox_live/index_test.exs

@@ -2907,6 +2907,72 @@ defmodule LightningWeb.SandboxLive.IndexTest do
       assert_redirect(view, ~p"/projects/#{parent.id}/w")
     end
 
+    test "commits to GitHub using YAML config when use_yaml_config is true",
+         %{
+           conn: conn,
+           parent: parent,
+           sandbox: sandbox,
+           snapshot: snapshot
+         } do
+      repo_connection =
+        insert(:project_repo_connection,
+          project: parent,
+          repo: "someaccount/somerepo",
+          branch: "main",
+          github_installation_id: "1234",
+          use_yaml_config: true
+        )
+
+      {:ok, view, _html} = live(conn, ~p"/projects/#{parent.id}/sandboxes")
+
+      expect_create_installation_token(repo_connection.github_installation_id)
+      expect_get_repo(repo_connection.repo)
+
+      expect_create_workflow_dispatch_with_request_body(
+        repo_connection.repo,
+        "openfn-pull.yml",
+        %{
+          ref: "main",
+          inputs: %{
+            projectId: parent.id,
+            apiSecretName: api_secret_name(parent),
+            branch: repo_connection.branch,
+            pathToConfig: path_to_config(repo_connection),
+            commitMessage: "pre-merge commit",
+            snapshots: "#{snapshot.id}"
+          }
+        }
+      )
+
+      expect_create_installation_token(repo_connection.github_installation_id)
+      expect_get_repo(repo_connection.repo)
+
+      expect_create_workflow_dispatch_with_request_body(
+        repo_connection.repo,
+        "openfn-pull.yml",
+        %{
+          ref: "main",
+          inputs: %{
+            projectId: parent.id,
+            apiSecretName: api_secret_name(parent),
+            branch: repo_connection.branch,
+            pathToConfig: path_to_config(repo_connection),
+            commitMessage: "Merged sandbox #{sandbox.name}"
+          }
+        }
+      )
+
+      view
+      |> element("#branch-rewire-sandbox-#{sandbox.id} button")
+      |> render_click()
+
+      view
+      |> form("#merge-sandbox-modal form")
+      |> render_submit()
+
+      assert_redirect(view, ~p"/projects/#{parent.id}/w")
+    end
+
     test "does not commit to GitHub when project has no GitHub sync configured",
          %{
            conn: conn,
@@ -2937,8 +3003,7 @@ defmodule LightningWeb.SandboxLive.IndexTest do
     end
 
     defp path_to_config(repo_connection) do
-      repo_connection
-      |> Lightning.VersionControl.ProjectRepoConnection.config_path()
+      Lightning.VersionControl.ProjectRepoConnection.config_path(repo_connection)
       |> Path.relative_to(".")
     end
   end