diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c1c2133a..7baef2182e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to ### Added +- Add support for sync v2 protocol + [#4523](https://github.com/OpenFn/lightning/issues/4523) - Support collections in sandboxes. Collection names are now scoped per project, empty collections are cloned into a sandbox on provision, and collection names (not data) are synchronised when a sandbox is merged back into its parent. The diff --git a/lib/lightning/version_control/project_repo_connection.ex b/lib/lightning/version_control/project_repo_connection.ex index f8b311f204..28b5a6da33 100644 --- a/lib/lightning/version_control/project_repo_connection.ex +++ b/lib/lightning/version_control/project_repo_connection.ex @@ -22,6 +22,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do field :branch, :string field :access_token, :binary field :config_path, :string + field :sync_version, :boolean, default: false field :accept, :boolean, virtual: true field :sync_direction, Ecto.Enum, @@ -56,7 +57,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do end @required_fields ~w(github_installation_id repo branch project_id)a - @other_fields ~w(config_path)a + @other_fields ~w(config_path sync_version)a def changeset(project_repo_connection, attrs) do project_repo_connection @@ -125,6 +126,14 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do def config_path(repo_connection) do repo_connection.config_path || - "./openfn-#{repo_connection.project_id}-config.json" + if repo_connection.sync_version do + openfn_yaml() + else + "./openfn-#{repo_connection.project_id}-config.json" + end + end + + def openfn_yaml do + "openfn.yaml" end end diff --git a/lib/lightning/version_control/version_control.ex b/lib/lightning/version_control/version_control.ex index 0ef87c1de6..e8fae76d53 100644 --- a/lib/lightning/version_control/version_control.ex +++ b/lib/lightning/version_control/version_control.ex @@ -11,6 +11,7 @@ defmodule Lightning.VersionControl do alias Ecto.Multi alias Lightning.Accounts.User alias Lightning.Extensions.UsageLimiting + alias Lightning.Projects.Project alias Lightning.Repo alias Lightning.VersionControl.Audit alias Lightning.VersionControl.Events @@ -572,8 +573,15 @@ defmodule Lightning.VersionControl do defp maybe_create_config_blob(tesla_client, repo_connection) do if is_nil(repo_connection.config_path) do + content = + if repo_connection.sync_version do + openfn_yaml(repo_connection) + else + config_json(repo_connection) + end + GithubClient.create_blob(tesla_client, repo_connection.repo, %{ - content: config_json(repo_connection) + content: content }) else {:ok, nil} @@ -625,6 +633,16 @@ defmodule Lightning.VersionControl do ) end + defp openfn_yaml(repo_connection) do + project = Repo.get!(Project, repo_connection.project_id) + + """ + project: + uuid: #{project.id} + endpoint: #{LightningWeb.Endpoint.url()} + """ + end + defp pull_yml_target_path do ".github/workflows/openfn-pull.yml" end diff --git a/lib/lightning_web/live/project_live/github_sync_component.ex b/lib/lightning_web/live/project_live/github_sync_component.ex index 31168121ad..b1521a64b0 100644 --- a/lib/lightning_web/live/project_live/github_sync_component.ex +++ b/lib/lightning_web/live/project_live/github_sync_component.ex @@ -683,6 +683,44 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do """ end + attr :form, :map, required: true + attr :project_id, :string, required: true + + defp config_format_toggle(assigns) do + ~H""" +
+
+ + <.icon + name="hero-chevron-right-mini" + class="h-4 w-4 transition-transform group-open:rotate-90" + /> Advanced: use new YAML config format + +
+ +

+ Only enable this if you want to use the new openfn.yaml + format instead of the legacy JSON config. +

+
+
+
+ """ + end + + defp sync_version?(form) do + form[:sync_version].value in [true, "true"] + end + attr :form, :map, required: true defp sync_order_radio(assigns) do @@ -738,9 +776,9 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do Import from GitHub (overwrite this project)

- If you already have config.json - and project.yaml - files tracked on GitHub and you want to overwrite + If you already have an openfn.yaml + (or legacy config.json) + tracked on GitHub and you want to overwrite this project on OpenFn, you can choose this advanced option.

@@ -794,9 +832,7 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
  • <.icon name="hero-document-plus" class="h-4 w-4" /> - ./openfn-{@project.id}-config.json -> {@form[ - :branch - ].value} + {config_filename(@form, @project.id)} -> {@form[:branch].value}
  • <% end %> @@ -806,4 +842,12 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do """ end + + defp config_filename(form, project_id) do + if sync_version?(form) do + ProjectRepoConnection.openfn_yaml() + else + "openfn-#{project_id}-config.json" + end + end end diff --git a/lib/lightning_web/live/project_live/github_sync_component.html.heex b/lib/lightning_web/live/project_live/github_sync_component.html.heex index 361826edb2..6a435b33d9 100644 --- a/lib/lightning_web/live/project_live/github_sync_component.html.heex +++ b/lib/lightning_web/live/project_live/github_sync_component.html.heex @@ -145,17 +145,22 @@
    + <.sync_order_radio form={f} /> +
    +
    <.input type="text" field={f[:config_path]} - label={"Path to config #{if f[:sync_direction].value == :deploy, do: "(required)", else: "(optional)"}"} + label="Path to config (required)" placeholder={"./openfn-#{@project.id}-config.json"} class="placeholder:italic placeholder:text-slate-400" />
    -
    - <.sync_order_radio form={f} /> -
    + <.config_format_toggle + :if={f[:sync_direction].value != :deploy} + form={f} + project_id={@project.id} + /> <%= if f[:branch].value do %> <.accept_checkbox project={@project} diff --git a/priv/repo/migrations/20260420114808_use_sync_v2.exs b/priv/repo/migrations/20260420114808_use_sync_v2.exs new file mode 100644 index 0000000000..01f2e6ea64 --- /dev/null +++ b/priv/repo/migrations/20260420114808_use_sync_v2.exs @@ -0,0 +1,9 @@ +defmodule Lightning.Repo.Migrations.UseSyncV2 do + use Ecto.Migration + + def change do + alter table(:project_repo_connections) do + add :sync_version, :boolean, null: false, default: false + end + end +end diff --git a/test/lightning/version_control_test.exs b/test/lightning/version_control_test.exs index b02f61f5fa..afbb12eabb 100644 --- a/test/lightning/version_control_test.exs +++ b/test/lightning/version_control_test.exs @@ -617,7 +617,7 @@ defmodule Lightning.VersionControlTest do VersionControl.initiate_sync(repo_connection, commit_message) end - test "creates GH workflow dispatch event", %{ + test "creates GH workflow dispatch event using JSON config (default)", %{ commit_message: commit_message, repo_connection: repo_connection, snapshots: [snapshot, other_snapshot] @@ -644,6 +644,39 @@ defmodule Lightning.VersionControlTest do assert :ok = VersionControl.initiate_sync(repo_connection, commit_message) end + test "creates GH workflow dispatch event using YAML config (sync_version: true)", + %{ + commit_message: commit_message, + repo_connection: repo_connection, + snapshots: [snapshot, other_snapshot] + } do + yaml_connection = + repo_connection + |> Ecto.Changeset.change(sync_version: true) + |> Lightning.Repo.update!() + + expect_create_installation_token(yaml_connection.github_installation_id) + expect_get_repo(yaml_connection.repo) + + expect_create_workflow_dispatch_with_request_body( + yaml_connection.repo, + "openfn-pull.yml", + %{ + ref: "main", + inputs: %{ + projectId: yaml_connection.project_id, + apiSecretName: api_secret_name(yaml_connection), + branch: yaml_connection.branch, + pathToConfig: path_to_config(yaml_connection), + commitMessage: commit_message, + snapshots: "#{other_snapshot.id} #{snapshot.id}" + } + } + ) + + assert :ok = VersionControl.initiate_sync(yaml_connection, commit_message) + end + defp api_secret_name(%{project_id: project_id}) do project_id |> String.replace("-", "_") @@ -651,8 +684,7 @@ defmodule Lightning.VersionControlTest do end defp path_to_config(repo_connection) do - repo_connection - |> ProjectRepoConnection.config_path() + ProjectRepoConnection.config_path(repo_connection) |> Path.relative_to(".") end end @@ -833,6 +865,123 @@ defmodule Lightning.VersionControlTest do end end + describe "config file blob content" do + setup do + Mox.verify_on_exit!() + + project = insert(:project) + user = user_with_valid_github_oauth() + + repo = "someaccount/somerepo" + branch = "somebranch" + installation_id = "1234" + + base_params = %{ + "project_id" => project.id, + "repo" => repo, + "branch" => branch, + "github_installation_id" => installation_id, + "sync_direction" => "pull", + "accept" => "true" + } + + expected_repo = %{"full_name" => repo, "default_branch" => "main"} + + {:ok, + project: project, + user: user, + repo: repo, + branch: branch, + installation_id: installation_id, + base_params: base_params, + expected_repo: expected_repo} + end + + defp setup_github_mocks(repo, expected_repo) do + expect_get_repo(repo, 200, expected_repo) + expect_create_blob(repo) + expect_get_commit(repo, expected_repo["default_branch"]) + expect_create_tree(repo) + expect_create_commit(repo) + expect_update_ref(repo, expected_repo["default_branch"]) + expect_create_blob(repo) + end + + test "pushes JSON config blob when sync_version is false (default)", %{ + project: project, + user: user, + repo: repo, + branch: branch, + installation_id: installation_id, + base_params: base_params, + expected_repo: expected_repo + } do + setup_github_mocks(repo, expected_repo) + + Mox.expect(Lightning.Tesla.Mock, :call, fn env, _opts -> + assert env.url == "https://api.github.com/repos/#{repo}/git/blobs" + body = Jason.decode!(env.body) + + assert body["content"] =~ + "\"statePath\": \"openfn-#{project.id}-state.json\"" + + assert body["content"] =~ + "\"specPath\": \"openfn-#{project.id}-spec.yaml\"" + + {:ok, %Tesla.Env{status: 201, body: %{"sha" => "3a0f8"}}} + end) + + secret_name = "OPENFN_#{String.replace(project.id, "-", "_")}_API_KEY" + expect_get_commit(repo, branch) + expect_create_tree(repo) + expect_create_commit(repo) + expect_update_ref(repo, branch) + expect_get_public_key(repo) + expect_create_repo_secret(repo, secret_name) + expect_create_installation_token(installation_id) + expect_get_repo(repo, 200, expected_repo) + expect_create_workflow_dispatch(repo, "openfn-pull.yml") + + assert {:ok, _} = + VersionControl.create_github_connection(base_params, user) + end + + test "pushes YAML config blob when sync_version is true", %{ + project: project, + user: user, + repo: repo, + branch: branch, + installation_id: installation_id, + base_params: base_params, + expected_repo: expected_repo + } do + setup_github_mocks(repo, expected_repo) + + Mox.expect(Lightning.Tesla.Mock, :call, fn env, _opts -> + assert env.url == "https://api.github.com/repos/#{repo}/git/blobs" + body = Jason.decode!(env.body) + assert body["content"] =~ "project:" + assert body["content"] =~ "uuid: #{project.id}" + assert body["content"] =~ LightningWeb.Endpoint.url() + {:ok, %Tesla.Env{status: 201, body: %{"sha" => "3a0f8"}}} + end) + + secret_name = "OPENFN_#{String.replace(project.id, "-", "_")}_API_KEY" + expect_get_commit(repo, branch) + expect_create_tree(repo) + expect_create_commit(repo) + expect_update_ref(repo, branch) + expect_get_public_key(repo) + expect_create_repo_secret(repo, secret_name) + expect_create_installation_token(installation_id) + expect_get_repo(repo, 200, expected_repo) + expect_create_workflow_dispatch(repo, "openfn-pull.yml") + + params = Map.put(base_params, "sync_version", "true") + assert {:ok, _} = VersionControl.create_github_connection(params, user) + end + end + defp user_with_valid_github_oauth do active_token = %{ "access_token" => "access-token", diff --git a/test/lightning_web/live/sandbox_live/index_test.exs b/test/lightning_web/live/sandbox_live/index_test.exs index 2163e501ae..98745c2ccf 100644 --- a/test/lightning_web/live/sandbox_live/index_test.exs +++ b/test/lightning_web/live/sandbox_live/index_test.exs @@ -3055,6 +3055,72 @@ defmodule LightningWeb.SandboxLive.IndexTest do assert_redirect(view, ~p"/projects/#{parent.id}/w") end + test "commits to GitHub using YAML config when sync_version is true", + %{ + conn: conn, + parent: parent, + sandbox: sandbox, + snapshot: snapshot + } do + repo_connection = + insert(:project_repo_connection, + project: parent, + repo: "someaccount/somerepo", + branch: "main", + github_installation_id: "1234", + sync_version: true + ) + + {:ok, view, _html} = live(conn, ~p"/projects/#{parent.id}/sandboxes") + + expect_create_installation_token(repo_connection.github_installation_id) + expect_get_repo(repo_connection.repo) + + expect_create_workflow_dispatch_with_request_body( + repo_connection.repo, + "openfn-pull.yml", + %{ + ref: "main", + inputs: %{ + projectId: parent.id, + apiSecretName: api_secret_name(parent), + branch: repo_connection.branch, + pathToConfig: path_to_config(repo_connection), + commitMessage: "pre-merge commit", + snapshots: "#{snapshot.id}" + } + } + ) + + expect_create_installation_token(repo_connection.github_installation_id) + expect_get_repo(repo_connection.repo) + + expect_create_workflow_dispatch_with_request_body( + repo_connection.repo, + "openfn-pull.yml", + %{ + ref: "main", + inputs: %{ + projectId: parent.id, + apiSecretName: api_secret_name(parent), + branch: repo_connection.branch, + pathToConfig: path_to_config(repo_connection), + commitMessage: "Merged sandbox #{sandbox.name}" + } + } + ) + + view + |> element("#branch-rewire-sandbox-#{sandbox.id} button") + |> render_click() + + view + |> form("#merge-sandbox-modal form") + |> render_submit() + + assert_redirect(view, ~p"/projects/#{parent.id}/w") + end + test "does not commit to GitHub when project has no GitHub sync configured", %{ conn: conn, @@ -3085,8 +3151,7 @@ defmodule LightningWeb.SandboxLive.IndexTest do end defp path_to_config(repo_connection) do - repo_connection - |> Lightning.VersionControl.ProjectRepoConnection.config_path() + Lightning.VersionControl.ProjectRepoConnection.config_path(repo_connection) |> Path.relative_to(".") end end