Gt1180 apel loading validator

docs/blackhole_db_setup.md

@@ -0,0 +1,84 @@
+## Steps to set up a blackhole database for local testing of the loading validator
+
+As part of the validator app, records are sent to a local instance of the apel
+loader class, to test if they load into an apel server database correctly.
+This is important as some errors in a record won't be detected by the syntax
+validator, but will still cause a record to fail to load.
+
+These records are loaded into a database with blackhole engined tables - these
+tables allow insert commands, but don't store any rows or data, as data is
+discarded on write. This allows complete checking that a record can be
+successfully loaded to a database, without having to deal with data being stored.
+
+The local instance of the apel loader class is within `monitoring/views.py`, and
+uses `monitoring/validatorSettings.py` to pull configuration settings from
+`monitoring/settings.ini` about the blackhole validator database.
+
+Steps to set up a blackhole-engined version of the apel server database:
+
+1. Ensure maraidb is started and enabled:
+    - `sudo su`
+    - `sudo systemctl start mariadb`
+    - `sudo systemctl enable mariadb`
+
+2. Login to mariadb with root:
+    - `mysql -u root -p`
+
+3. Install the blackhole plugin, and then verify it is installed:
+    - `INSTALL SONAME 'ha_blackhole';`
+    - `SHOW ENGINES;` (should be a row with BLACKHOLE and support as YES).
+
+4. Exit mariadb:
+    - `exit;`
+
+5. Set the global default storage engine to blackhole, so that when the database
+    schema gets applied, tables are created with the blackhole engine:
+    - find where your mariadb settings are stored (for me it was `/etc/my.cnf.d/`).
+    - either edit `server.cnf` or create a new `blackhole.cnf` file (what I did).
+    - in that file:
+    ```
+    [mysqld]
+    default-storage-engine=BLACKHOLE
+    ```
+    - This config setting means that any create table statements without an engine
+    defined will be set to a blackhole engine by default.
+
+6. Restart mariadb:
+    - `sudo systemctl restart mariadb`
+    - Running `SHOW ENGINES;` within mariadb at this point should show the Blackhole
+    row with support as DEFAULT;
+
+7. Create the database:
+    - `mysql -u root -p`
+    - `CREATE DATABASE validator_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;`
+    - `CREATE USER 'your_name'@'localhost' IDENTIFIED BY 'your_password';`
+    - `GRANT ALL PRIVILEGES ON validator_db.* TO 'your_name'@'localhost';`
+    - `FLUSH PRIVILEGES;`
+    - `exit;`
+
+8. Apply the apel server schema to the database:
+    - the schema is at https://github.com/apel/apel/blob/dev/schemas/server.sql.
+    - to do this step, I used my locally cloned version of apel as the
+    schema file path.
+    - `mysql -u root validator_db < path_to_apel/schemas/server.sql`
+
+9. Verify the schema applied correctly, and that the correct tables use a
+    blackhole engine:
+    - `mysql -u your_name -p`
+    - `SHOW DATABASES;`
+    - `USE validator_db;`
+    - `SHOW TABLES;` (check all tables are there)
+    - `SHOW TABLE STATUS;` (check that all tables either have a BLACKHOLE or
+    NULL engine)
+
+10. Populate settings.ini with the following, adding in the correct values:
+    - ```
+      [db_validator]
+      backend=mysql
+      hostname=localhost
+      name=validator_db
+      password=
+      port=3306
+      username=
+      ```
+    - these config options are picked up by the `validatorSettings.py` file.

monitoring/validator/templates/validator/validator_index.html

@@ -41,18 +41,28 @@ <h1>Record Validator</h2>
             wrap="wrap"
           >{{ input_record|default:''|escape }}</textarea>
           <br><br>
-          <button type="submit" id="validate-button">Validate!</button>
+          <p>Select 'Validate' if you want to check only the formatting of the record(s).</p>
+          <p>Select 'Load' if you want to also check whether the record(s) will successfully load into the database.</p>
+          <input type="radio" name="submission_type" value="validate"> Validate
+          <br>
+          <input type="radio" name="submission_type" value="load"> Load
+          <br><br><br>
+          <button type="submit" id="validate-button">Submit Record(s)!</button>
           <br><br>
         </form>
 
         <script>
-          // Setting select option back to what was selected (as it gets reset on form submission)
+          // Setting select and radio option back to what was selected (as they get reset on form submission)
           document.getElementById("record_type").value = "{{ record_type }}";
+          if ("{{ submission_type }}" != "") {
+              const checkRadio = document.querySelector(`input[name=submission_type][value="{{ submission_type }}"]`);
+              checkRadio.checked = true;
+          }
         </script>
 
         <div id="output">
           {% if output %}
-            <h3>Validation Output</h3>
+            <h3>Submission Output</h3>
             {{ output }}
           {% endif %}
         </div>

monitoring/validator/views.py

@@ -1,7 +1,9 @@
 from django.shortcuts import render
 from django.views.decorators.http import require_http_methods
 
-# Apel record-checking class imports
+# Apel loader and record-checking class imports
+from apel.db import ApelDbException
+from apel.db.loader.loader import Loader, LoaderException
 from apel.db.loader.record_factory import RecordFactory, RecordFactoryException
 from apel.db.records.record import InvalidRecordException
 
@@ -13,30 +15,56 @@
 from apel.db.records.cloud import CloudRecord
 from apel.db.records.cloud_summary import CloudSummaryRecord
 
+# Validator db configuration
+from monitoring import validatorSettings
+
+# Python tempfile for temporary apel queues
+import tempfile
+
 
 @require_http_methods(["GET", "POST"])
 def index(request):
     """
-    Validates inputted records using the Apel record validation methods.
-    It either validates a record against a specific type or against all types, depending on what record_type
+    Validates inputted records using the Apel record validation methods, or tests they load correctly using an
+        instance of the Apel loader.
+    For validation:
+        It either validates a record against a specific type or against all types, depending on what record_type
         option was chosen on the html template. The default is `All`.
-    The input record, record type and validation output are then returned to the html template as context on get
-        request, so that the html page retains its information/context when refreshing the page or submitting the form.
+    For loading:
+        It both validates the record's syntax against all types, and checks it can load into a database correctly
+        using the Apel loader class. The database uses a blackhole engine, so no data is stored.
+    The input record, record type, submission type and validation output are then returned to the html template
+        as context on get request, so that the html page retains its information/context when refreshing the page
+        or submitting the form.
     """
+
     template_name = "validator/validator_index.html"
     input_record = ""
     record_type = "All"
+    submission_type = ""
     output = ""
 
-    # On form submission, trigger record validation
+    # On form submission, check record isnt empty.
+    # Then trigger record validation or record loading, based on submission type
     if request.method == "POST":
         input_record = request.POST.get("input_record", "")
         record_type = request.POST.get("record_type", "")
-        output = validate(input_record, record_type)
+        submission_type = request.POST.get("submission_type", "")
+
+        if input_record:
+            input_record = input_record.strip()
+
+            if submission_type == "load":
+                output = load(input_record)
+            else:
+                output = validate(input_record, record_type)
+        else:
+            output = "Please enter a record to be validated."
 
     context = {
         "input_record": input_record,
         "record_type": record_type,
+        "submission_type": submission_type,
         "output": output,
     }
 
@@ -45,17 +73,13 @@ def index(request):
 
 def validate(record: str, record_type: str) -> str:
     """
-    Validated record(s) and record_type passed in from the html page template.
-    If record type is all, make use of the create_records apel method (expects a record header).
-    Else, make use of the _create_record_objects apel method (expects there to be no record header).
+    Record(s) and record_type passed in from the html page template.
+    If record type is all, make use of the `create_records` apel method (expects a record header).
+    Else, make use of the `_create_record_objects` apel method (expects there to be no record header).
     If the record is valid, return a "valid record" string.
     If the record is invalid, an InvalidRecordException or RecordFactoryException is raised by the Apel methods.
     Catch these exceptions and return the exception information.
     """
-    if not record:
-        return "Please enter a record to be validated."
-
-    record = record.strip()
 
     # Map record_type string to record_type class
     # String is always exact as determined through html form selection option
@@ -80,10 +104,50 @@ def validate(record: str, record_type: str) -> str:
             record_class = record_map[record_type]
             result = recordFactory._create_record_objects(record, record_class)
 
-        if "Record object at" in str(result):
+        if "object at" in str(result):
             return "Record(s) valid!"
 
         return str(result)
 
     except (InvalidRecordException, RecordFactoryException) as e:
         return str(e)
+
+def load(record: str) -> str:
+    """
+    Record passed in from the html page template.
+    Create a loader instance, making use of a tempfile directory for the queues and pidfile creation
+    Startup loader and then pass record(s) and blank signer into the `load_msg` method
+    Make use of a blackhole database, which loads but doesn't store any data.
+    If the record loads successfully, return a "valid record" string.
+    If the record fails to load, an exception is raised by the Apel methods.
+    Catch these exceptions and return the exception information.
+    """
+
+    try:
+        validatorDB = validatorSettings.VALIDATOR_DB
+
+        # Set the tempfile temporary directory within /tmp
+        tempfile.tempdir = "/tmp"
+
+        qpath = tempfile.gettempdir()
+        db_backend = validatorDB.get("ENGINE")
+        db_host = validatorDB.get("HOST")
+        db_port = int(validatorDB.get("PORT"))
+        db_name = validatorDB.get("NAME")
+        db_username = validatorDB.get("USER")
+        db_password = validatorDB.get("PASSWORD")
+        pidfile = ""
+        signer = ""
+
+        loader = Loader(qpath, record, db_backend, db_host, db_port, db_name, db_username, db_password, pidfile)
+
+        loader.startup()
+
+        loader.load_msg(record, signer)
+
+        loader.shutdown()
+
+        return("Record(s) will load successfully!")
+
+    except (ApelDbException, InvalidRecordException, LoaderException, RecordFactoryException) as e:
+        return str(e)

monitoring/validatorSettings.py

@@ -0,0 +1,33 @@
+"""
+Settings for the validator app, part of the monitoring project.
+
+These settings are for the apel modules, so they need to be kept separate
+from Django's control. Because Django was having issues with some of
+the defined configuration, such as the database engine.
+"""
+
+import configparser
+import os
+import sys
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    # Read configuration from the file
+    cp = configparser.ConfigParser(interpolation=None)
+    file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini')
+    cp.read(file_path)
+
+    VALIDATOR_DB = {
+        'ENGINE': cp.get('db_validator', 'backend'),
+        'HOST': cp.get('db_validator', 'hostname'),
+        'PORT': cp.get('db_validator', 'port'),
+        'NAME': cp.get('db_validator', 'name'),
+        'USER': cp.get('db_validator', 'username'),
+        'PASSWORD': cp.get('db_validator', 'password'),
+    }
+
+except (configparser.NoSectionError) as err:
+    print("Error in configuration file. Check that file exists first: %s" % err)
+    sys.exit(1)

requirements.txt

@@ -1,5 +1,4 @@
 # Pin packages to support and work with py3.6.
-apel
 Django==3.2.25
 djangorestframework==3.15.1
 pytz==2025.2