diff --git a/CHANGELOG.md b/CHANGELOG.md
index 71c1c2133a..7baef2182e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ and this project adheres to
### Added
+- Add support for sync v2 protocol
+ [#4523](https://github.com/OpenFn/lightning/issues/4523)
- Support collections in sandboxes. Collection names are now scoped per project,
empty collections are cloned into a sandbox on provision, and collection names
(not data) are synchronised when a sandbox is merged back into its parent. The
diff --git a/lib/lightning/version_control/project_repo_connection.ex b/lib/lightning/version_control/project_repo_connection.ex
index f8b311f204..28b5a6da33 100644
--- a/lib/lightning/version_control/project_repo_connection.ex
+++ b/lib/lightning/version_control/project_repo_connection.ex
@@ -22,6 +22,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
field :branch, :string
field :access_token, :binary
field :config_path, :string
+ field :sync_version, :boolean, default: false
field :accept, :boolean, virtual: true
field :sync_direction, Ecto.Enum,
@@ -56,7 +57,7 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
end
@required_fields ~w(github_installation_id repo branch project_id)a
- @other_fields ~w(config_path)a
+ @other_fields ~w(config_path sync_version)a
def changeset(project_repo_connection, attrs) do
project_repo_connection
@@ -125,6 +126,14 @@ defmodule Lightning.VersionControl.ProjectRepoConnection do
def config_path(repo_connection) do
repo_connection.config_path ||
- "./openfn-#{repo_connection.project_id}-config.json"
+ if repo_connection.sync_version do
+ openfn_yaml()
+ else
+ "./openfn-#{repo_connection.project_id}-config.json"
+ end
+ end
+
+ def openfn_yaml do
+ "openfn.yaml"
end
end
diff --git a/lib/lightning/version_control/version_control.ex b/lib/lightning/version_control/version_control.ex
index 0ef87c1de6..e8fae76d53 100644
--- a/lib/lightning/version_control/version_control.ex
+++ b/lib/lightning/version_control/version_control.ex
@@ -11,6 +11,7 @@ defmodule Lightning.VersionControl do
alias Ecto.Multi
alias Lightning.Accounts.User
alias Lightning.Extensions.UsageLimiting
+ alias Lightning.Projects.Project
alias Lightning.Repo
alias Lightning.VersionControl.Audit
alias Lightning.VersionControl.Events
@@ -572,8 +573,15 @@ defmodule Lightning.VersionControl do
defp maybe_create_config_blob(tesla_client, repo_connection) do
if is_nil(repo_connection.config_path) do
+ content =
+ if repo_connection.sync_version do
+ openfn_yaml(repo_connection)
+ else
+ config_json(repo_connection)
+ end
+
GithubClient.create_blob(tesla_client, repo_connection.repo, %{
- content: config_json(repo_connection)
+ content: content
})
else
{:ok, nil}
@@ -625,6 +633,16 @@ defmodule Lightning.VersionControl do
)
end
+ defp openfn_yaml(repo_connection) do
+ project = Repo.get!(Project, repo_connection.project_id)
+
+ """
+ project:
+ uuid: #{project.id}
+ endpoint: #{LightningWeb.Endpoint.url()}
+ """
+ end
+
defp pull_yml_target_path do
".github/workflows/openfn-pull.yml"
end
diff --git a/lib/lightning_web/live/project_live/github_sync_component.ex b/lib/lightning_web/live/project_live/github_sync_component.ex
index 31168121ad..b1521a64b0 100644
--- a/lib/lightning_web/live/project_live/github_sync_component.ex
+++ b/lib/lightning_web/live/project_live/github_sync_component.ex
@@ -683,6 +683,44 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
"""
end
+ attr :form, :map, required: true
+ attr :project_id, :string, required: true
+
+ defp config_format_toggle(assigns) do
+ ~H"""
+
+
+
+ <.icon
+ name="hero-chevron-right-mini"
+ class="h-4 w-4 transition-transform group-open:rotate-90"
+ /> Advanced: use new YAML config format
+
+
+
+
+ Only enable this if you want to use the new openfn.yaml
+ format instead of the legacy JSON config.
+
+
+
+
+ """
+ end
+
+ defp sync_version?(form) do
+ form[:sync_version].value in [true, "true"]
+ end
+
attr :form, :map, required: true
defp sync_order_radio(assigns) do
@@ -738,9 +776,9 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
Import from GitHub (overwrite this project)
- If you already have config.json
- and project.yaml
- files tracked on GitHub and you want to overwrite
+ If you already have an openfn.yaml
+ (or legacy config.json)
+ tracked on GitHub and you want to overwrite
this project on OpenFn, you can choose this advanced option.
@@ -794,9 +832,7 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
<% end %>
@@ -806,4 +842,12 @@ defmodule LightningWeb.ProjectLive.GithubSyncComponent do
"""
end
+
+ defp config_filename(form, project_id) do
+ if sync_version?(form) do
+ ProjectRepoConnection.openfn_yaml()
+ else
+ "openfn-#{project_id}-config.json"
+ end
+ end
end
diff --git a/lib/lightning_web/live/project_live/github_sync_component.html.heex b/lib/lightning_web/live/project_live/github_sync_component.html.heex
index 361826edb2..6a435b33d9 100644
--- a/lib/lightning_web/live/project_live/github_sync_component.html.heex
+++ b/lib/lightning_web/live/project_live/github_sync_component.html.heex
@@ -145,17 +145,22 @@