feat: dmtri fix — automated repair of consistency-report inconsistencies

.gitignore

@@ -29,5 +29,18 @@ src/*.egg-info/
 # User/Agent Documentation
 JOBS_DOCUMENTATION.md
 AGENTS.md
+reports/
+
+# AI assistant folders / files
 .antigravity/
-reports/
+.claude/
+.cursor/
+.windsurf/
+.aider*
+CLAUDE.md
+GEMINI.md
+.clinerules/
+
+# Demo recordings (asciinema casts and rendered GIFs)
+*.cast
+*.gif

README.md

@@ -74,6 +74,45 @@ Monitor the status of asynchronous jobs:
 dmtri track
 ```
 
+### Fix Inconsistencies
+Automatically repair the inconsistencies found in an [`eida-consistency`](https://github.com/EIDA/eida-consistency) report. A report lists streams where the **availability** view and **dataselect** disagree; `dmtri fix` reads that report and, for each affected stream, re-runs the WFCatalog collector and rebuilds the availability view for the exact time window.
+
+Point it at a report file or URL:
+
+```bash
+dmtri fix https://eida-oculus.orfeus-eu.org/consistency/NOA/2026/NOA_2026-06-07_140510.json
+```
+
+Fix only specific entries from the report (by their index):
+
+```bash
+dmtri fix report.json --index 2 --index 16
+```
+
+After a fix, the availability service keeps cached answers for ~20 minutes, so a stream may still look unfixed for a short while. Re-check once the cache expires — this only verifies, it changes nothing:
+
+```bash
+dmtri fix --verify-only report.json
+```
+
+#### Prerequisite: the `eida-consistency` CLI
+
+`dmtri fix` reads the report through [`eida-consistency`](https://github.com/EIDA/eida-consistency) (version **0.5.1 or newer**, for its `explore --json` output). If it isn't already on the machine, install it any one of these ways:
+
+```bash
+uv tool install eida-consistency     # recommended — puts it on your PATH
+pipx install eida-consistency
+pip install eida-consistency
+```
+
+You don't strictly have to install it: if `uv` is present, `dmtri fix` will run it on demand via `uvx eida-consistency`. And if it's installed somewhere off your `PATH`, point dmtri at it:
+
+```bash
+export DMTRI_EIDA_CONSISTENCY='/full/path/to/eida-consistency'
+```
+
+If it's missing (or too old), `dmtri fix` stops before touching anything and prints the exact install/upgrade command to run.
+
 ---
 
 ## Customization

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
-name = "eida-dmtri"
-version = "0.1.0"
+name = "dmtri"
+version = "0.1.1"
 description = "CLI tool for triggering datacenter metadata updates"
 authors = [{ name = "Nikos Sokos", email = "nsokos@noa.com" }]
 dependencies = [

src/dmtri/cli.py

@@ -53,6 +53,14 @@ def main():
 
     subparsers.add_parser("refresh", parents=[shared], help="Refresh data/metadata using configured playbooks.")
     subparsers.add_parser("clean", parents=[shared], help="Clean outdated data/metadata using configured playbooks.")
+
+    fix_parser = subparsers.add_parser("fix", help="Automatically repair inconsistencies from an eida-consistency report.")
+    fix_parser.add_argument("report", help="Consistency report: URL or local JSON path")
+    fix_parser.add_argument("--verify-only", action="store_true", help="Only re-check status (no rebuilding)")
+    fix_parser.add_argument("--days", type=int, default=None, help="Max days to explore boundaries (passthrough to eida-consistency)")
+    fix_parser.add_argument("--index", type=int, action="append", dest="index", help="Restrict to specific report index(es); repeatable")
+    fix_parser.add_argument("--no-confirm", action="store_true", help="Skip confirmation prompt before executing")
+    fix_parser.add_argument("--debug", action="store_true", help="Show verbose output from Ansible")
     subparsers.add_parser("track", help="Track job status via tracking playbook.")
     doctor_parser = subparsers.add_parser("doctor", help="Check SSH connectivity to all inventory hosts")
     doctor_parser.add_argument("-v", "--verbose", action="store_true", help="Show full Ansible output per host")
@@ -122,6 +130,20 @@ def main():
         for pb in playbooks:
             run_hook(pb, vars_to_pass, inventory=inventory_path)
 
+    elif args.command == "fix":
+        from dmtri.fix import run_fix, run_verify_only
+        try:
+            if args.verify_only:
+                rc = run_verify_only(args.report, inventory=inventory_path,
+                                     days=args.days, indices=args.index)
+            else:
+                rc = run_fix(args.report, inventory=inventory_path, days=args.days,
+                             indices=args.index, no_confirm=args.no_confirm, debug=args.debug)
+        except RuntimeError as e:
+            print(f"Error: {e}", file=sys.stderr)
+            sys.exit(2)
+        sys.exit(rc)
+
     elif args.command == "track":
         playbooks = COMMAND_PLAYBOOKS.get("track", [])
         if not playbooks:

src/dmtri/data/playbooks/fix/availability_rebuild.yml

@@ -0,0 +1,89 @@
+- name: "Rebuild availability materialized view for a specific NSLC + date range"
+  hosts: ws_availability
+  gather_facts: false
+
+  # Expected extra-vars (one stream, one range — dmtri fix calls this once per fix):
+  #   net, sta, cha   : exact codes
+  #   loc             : location code; pass "--" for the empty location
+  #   start, end      : YYYY-MM-DD. `end` is exclusive at the cacher (te <= end),
+  #                     so dmtri passes the day AFTER the last broken day.
+  #
+  # Unlike availability_refresh.yml (a global `docker restart` = last-24h rebuild),
+  # this triggers the in-container `avail-rebuild` console script, which rebuilds
+  # only the given stream/range from WFCatalog (idempotent $merge).
+
+  tasks:
+
+    # command + argv (not shell): values are passed as separate execve arguments,
+    # so even if a report-derived value contained shell metacharacters it cannot be
+    # interpreted by a shell. dmtri also validates these against strict allow-lists
+    # before they ever reach here (see fix._validate_fix).
+    - name: "Run scoped availability rebuild (docker exec avail-rebuild)"
+      command:
+        argv:
+          - docker
+          - exec
+          - fdsnws-availability-cacher
+          - avail-rebuild
+          - --net
+          - "{{ net }}"
+          - --sta
+          - "{{ sta }}"
+          - "--loc={{ loc }}"
+          - --cha
+          - "{{ cha }}"
+          - --start
+          - "{{ start }}"
+          - --end
+          - "{{ end }}"
+      register: rebuild_result
+
+    - name: "DISPLAY REBUILD STATUS"
+      debug:
+        msg:
+          - "========================================"
+          - " Availability Rebuild — {{ net }}.{{ sta }}.{{ loc }}.{{ cha }}"
+          - "========================================"
+          - "  Range    : {{ start }} -> {{ end }} (end exclusive)"
+          - "  Status   : {{ 'SUCCESS' if rebuild_result.rc == 0 else 'FAILED' }}"
+          - "  Exit Code: {{ rebuild_result.rc }}"
+          - "========================================"
+
+    # ---------------------------------------------------------------
+    # JOB TRACKING
+    # ---------------------------------------------------------------
+
+    - name: Ensure ~/.dmtri/jobs/{{ inventory_hostname }} directory exists
+      file:
+        path: "~/.dmtri/jobs/{{ inventory_hostname }}"
+        state: directory
+        mode: '0755'
+
+    - name: "Load existing job list (if exists)"
+      slurp:
+        src: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json"
+      register: job_file
+      ignore_errors: true
+
+    - name: "Parse job list or default to []"
+      set_fact:
+        job_list: >-
+          {{ (job_file.content | b64decode | from_json) if job_file.content is defined else [] }}
+
+    - name: Define new job entry
+      set_fact:
+        new_job: {
+          "type": "availability_rebuild",
+          "job_id": "rebuild-{{ lookup('pipe', 'date +%s') }}",
+          "host": "{{ inventory_hostname }}",
+          "stream": "{{ net }}.{{ sta }}.{{ loc }}.{{ cha }}",
+          "start": "{{ start }}",
+          "end": "{{ end }}",
+          "rc": "{{ rebuild_result.rc | default('unknown') }}",
+          "created_at": "{{ lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ') }}"
+        }
+
+    - name: "Save updated job list"
+      copy:
+        content: "{{ (job_list + [new_job]) | to_nice_json }}"
+        dest: "~/.dmtri/jobs/{{ inventory_hostname }}/availability.json"