schema: Use vkprofiles.exe to generate the schema

CHANGELOG.md

@@ -15,6 +15,8 @@
 
 ### Features:
 - Add `vkprofiles` executable which bundles the python scripts in a standalone executable
+  - Validate profiles JSON file with `validate` command
+  - Generate profiles schema file with `schema` command
 
 ### Improvements:
 - Improve profiles schema to support capabilities dynamic structures

CMakeLists.txt

@@ -104,7 +104,7 @@ function(add_pyinstaller_target TARGET_NAME)
     # Define the arguments the function accepts
     set(options)
     set(oneValueArgs SCRIPT OUTPUT_DIR FOLDER OUTPUT_NAME)
-    set(multiValueArgs DEPENDS)
+    set(multiValueArgs DEPENDS PATHS)
     cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
     if(NOT ARG_SCRIPT)
@@ -127,10 +127,15 @@ function(add_pyinstaller_target TARGET_NAME)
     endif()
     set(FINAL_EXECUTABLE "${ARG_OUTPUT_DIR}/${EXE_NAME}")
 
+    set(PYINSTALLER_PATH_FLAGS)
+    foreach(p ${ARG_PATHS})
+        list(APPEND PYINSTALLER_PATH_FLAGS "--paths" "${p}")
+    endforeach()
+
     # Generate the executable via PyInstaller using the VENV python executable
     add_custom_command(
         OUTPUT "${FINAL_EXECUTABLE}"
-        COMMAND "${VENV_PYTHON_EXECUTABLE}" -m PyInstaller --onefile --name "${ARG_OUTPUT_NAME}" --distpath "${ARG_OUTPUT_DIR}" "${ARG_SCRIPT}"
+        COMMAND "${VENV_PYTHON_EXECUTABLE}" -m PyInstaller --onefile --clean ${PYINSTALLER_PATH_FLAGS} --name "${ARG_OUTPUT_NAME}" --distpath "${ARG_OUTPUT_DIR}" "${ARG_SCRIPT}"
         DEPENDS ${ARG_DEPENDS} python_venv
         WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
         COMMENT "Packaging ${TARGET_NAME} into a standalone executable..."
@@ -161,10 +166,22 @@ set(PYTHON_DEPENDENCIES
     "${CMAKE_CURRENT_SOURCE_DIR}/scripts/source/vulkan_object_version.py"
 )
 
+if(VULKAN_HEADERS_INSTALL_DIR)
+    set(VULKAN_REGISTRY_DIR "${VULKAN_HEADERS_INSTALL_DIR}/registry")
+    set(VULKAN_REGISTRY_SHARE_DIR "${VULKAN_HEADERS_INSTALL_DIR}/share/vulkan/registry")
+elseif(TARGET Vulkan::Registry)
+    get_target_property(VULKAN_REGISTRY_DIR Vulkan::Registry INTERFACE_INCLUDE_DIRECTORIES)
+    set(VULKAN_REGISTRY_SHARE_DIR "${VULKAN_REGISTRY_DIR}")
+endif()
+
 add_pyinstaller_target(VpProfilesProcessor
     OUTPUT_NAME "vkprofiles"
     SCRIPT "${PROFILES_SCRIPT}"
     OUTPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts"
+    PATHS
+        "${VULKAN_REGISTRY_DIR}"
+        "${VULKAN_REGISTRY_SHARE_DIR}"
+        "${CMAKE_CURRENT_SOURCE_DIR}/scripts/source"
     DEPENDS ${PYTHON_DEPENDENCIES}
     FOLDER "Profiles generator"
 )

profiles/CMakeLists.txt

@@ -49,19 +49,33 @@ set(PROFILES_FILES_FOR_ANDROID_DOC
 )
 
 # Generate profiles schema
+if(WIN32)
+    set(VKPROFILES_EXE "${PROJECT_SOURCE_DIR}/scripts/vkprofiles.exe")
+else()
+    set(VKPROFILES_EXE "${PROJECT_SOURCE_DIR}/scripts/vkprofiles")
+endif()
+
 add_custom_target(VpGenerate-ProfilesSchema
     COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_SOURCE_DIR}/schema
-    COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${VULKAN_HEADERS_INSTALL_DIR}/${CMAKE_INSTALL_DATADIR}/vulkan/registry"
-        ${VENV_PYTHON_EXECUTABLE} ${SOLUTION_SCRIPT}
-        --api ${API_TYPE}
+    COMMAND ${VKPROFILES_EXE}
+        schema
         --registry ${VULKAN_HEADERS_INSTALL_DIR}/${CMAKE_INSTALL_DATADIR}/vulkan/registry/vk.xml
-        --output-schema ${PROJECT_SOURCE_DIR}/schema/${PROFILES_SCHEMA_FILENAME}
-        --validate
-    VERBATIM
-    SOURCES ${SOLUTION_SCRIPT}
-    DEPENDS ${SOLUTION_SCRIPT} python_venv)
+        --output ${PROJECT_SOURCE_DIR}/schema/${PROFILES_SCHEMA_FILENAME}
+        --api ${API_TYPE}
+    VERBATIM)
+
+#    COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${VULKAN_HEADERS_INSTALL_DIR}/${CMAKE_INSTALL_DATADIR}/vulkan/registry"
+#        ${VENV_PYTHON_EXECUTABLE} ${SOLUTION_SCRIPT}
+#        --api ${API_TYPE}
+#        --registry ${VULKAN_HEADERS_INSTALL_DIR}/${CMAKE_INSTALL_DATADIR}/vulkan/registry/vk.xml
+#        --output-schema ${PROJECT_SOURCE_DIR}/schema/${PROFILES_SCHEMA_FILENAME}
+#        --validate
+#    VERBATIM
+#    SOURCES ${SOLUTION_SCRIPT}
+#    DEPENDS ${SOLUTION_SCRIPT} python_venv)
+
 set_target_properties(VpGenerate-ProfilesSchema PROPERTIES FOLDER "Profiles schema")
-add_dependencies(VpGenerate-ProfilesSchema VpProfilesProcessor)
+add_dependencies(VpGenerate-ProfilesSchema VpProfilesProcessor python_venv)
 
 set(PROFILE_DESKTOP_MAX_2024_LABEL "LunarG Vulkan Desktop Max 2024 profile")
 set(PROFILE_DESKTOP_MAX_2024_DESC "A profile generated by the intersection of a collection of GPUInfo.org device reports to support the latest AMD, Intel and NVIDIA GPUs and drivers.")

scripts/gen_profiles_solution.py

@@ -5694,7 +5694,7 @@ def gen_videoFormats(self, videoProfileName, videoCodec):
 
     if args.registry != None:
         registry = VulkanRegistry(args.registry, args.api)
-        vk: VulkanObject = initVulkanObject(args.registry, True)
+        vk: VulkanObject = initVulkanObject(args.api, args.registry, True)
 
     if args.output_schema != None or args.validate:
         generator = VulkanProfilesSchemaGenerator(registry)

scripts/profiles.py

@@ -21,18 +21,20 @@
 
 import logging
 from datetime import datetime
-from enum import StrEnum
+from enum import Enum
 from pathlib import Path
 import argparse
 import sys
 from vulkan_object import VulkanObject
 from source.vulkan_object_utils import initVulkanObject, VK_VERSION, gatherDependentExtensions
 from source.profiles_parsing import load_profiles_jsons
 from source.profiles_parsing import save_profiles_jsons
-from source.profiles_parsing import validate_profiles_json
+from source.profiles_parsing import validate_profiles_json, validate_profiles_jsons_data
 from source.profiles_parsing import OutputFormatType
+from source.generate_profiles_schema import VulkanProfilesSchemaGenerator2
+from source.log import Log
 
-class ConvertMode(StrEnum):
+class ConvertMode(str, Enum):
     STRIP_DUPLICATION = 'strip-duplication'
     PULL_DEPENDENCES = 'pull-dependences'
 
@@ -123,7 +125,7 @@ def strip_profiles_files_capabilities_duplication(json_files_dict):
 
 
 def main_convert(args):
-    vk = initVulkanObject(args.registry or None)
+    vk = initVulkanObject(args.api, args.registry or None)
 
     for version in vk.versions.values():
         logging.debug(version.name)
@@ -151,8 +153,25 @@ def main_convert(args):
 
 
 def main_validate(args):
-    validate_profiles_json(Path(args.input), Path(args.schema))
+    if args.schema is None:
+        if args.registry is None:
+            Log.e("`--schema` or `--registry` are required to validate profile files")
+        else:
+            vk: VulkanObject = initVulkanObject(args.api, args.registry, True)
+            generator2 = VulkanProfilesSchemaGenerator2(vk)
+            validate_profiles_jsons_data(Path(args.input), generator2.schema)
+    else:
+        validate_profiles_json(Path(args.input), Path(args.schema))
+
 
+def main_schema(args):
+    vk: VulkanObject
+    if args.registry is None:
+        vk = initVulkanObject(args.api)
+    else:
+        vk = initVulkanObject(args.api, args.registry, True)
+    generator = VulkanProfilesSchemaGenerator2(vk)
+    generator.generate(args.output)
 
 def main(argv):
     logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')
@@ -170,15 +189,23 @@ def main(argv):
     convert_parser.add_argument('--mode', nargs='*',action='store', choices=list(ConvertMode), default=list(ConvertMode), help='List of conversion capabilities')
 
     validate_parser = subparsers.add_parser('validate', help='Validate a profile file against a profile schema.')
-    validate_parser.add_argument('--schema', '-s', action='store', required=True, help='Use a specific Vulkan registry file (vk.xml).')
+    validate_parser.add_argument('--registry', '-r', action='store', help='Use a specific Vulkan registry file (vk.xml).')
+    validate_parser.add_argument('--schema', '-s', action='store', help='Use a profile schema (profiles-*.json). By default, generate a profile schema vk.xml.')
     validate_parser.add_argument('--input', '-i', action='store', required=True, help='Path to the input profiles files.')
 
+    schema_parser = subparsers.add_parser('schema', help='Generate a profile json schema file.')
+    schema_parser.add_argument('--registry', '-r', action='store', help='Use a specific Vulkan registry file (vk.xml).')
+    schema_parser.add_argument('--output', '-o', action='store', required=True, help='Path to the output profile schema file.')
+    schema_parser.add_argument('--api', action='store', default='vulkan', choices=['vulkan'], help="Target API")
+
     args = parser.parse_args(argv)
 
     if args.command == 'convert':
         main_convert(args)
     elif args.command == 'validate':
         main_validate(args)
+    elif args.command == 'schema':
+        main_schema(args)
     else:
         parser.print_help()
 

scripts/source/generate_profiles_schema.py

@@ -40,7 +40,6 @@ def __init__(self, vk):
 
         # Call the global discovery helper passing the VulkanObject parameter
         self.valid_dynamic_structs = gatherDynamicStructs(vk)
-
         self.schema = self.gen_schema()
 
     def validate(self):

scripts/source/profiles_parsing.py

@@ -43,6 +43,7 @@ def _validate_profiles_json_data(json_data, schema_data) -> bool:
         logging.warning("`jsonschema` module is not installed, schema validation skip")
         return False
 
+
 def validate_profiles_json(json_data_path: Path, json_schema_path: Path) -> bool:
     schema_data = load_schema_json(json_schema_path)
     if schema_data is None:
@@ -59,12 +60,7 @@ def validate_profiles_json(json_data_path: Path, json_schema_path: Path) -> bool
     return _validate_profiles_json_data(json_data, schema_data)
 
 
-def validate_profiles_jsons(json_data_dir: Path, json_schema_path: Path) -> int:
-    schema_data = load_schema_json(json_schema_path)
-    if schema_data is None:
-        logging.error(f"Invalid profile file: {json_schema_path}")
-        return 0
-
+def validate_profiles_jsons_data(json_data_dir: Path, json_schema_data) -> int:
     profiles_files_paths = []
     for pos_json in os.listdir(json_data_dir):
         if pos_json.endswith('.json'):
@@ -78,12 +74,21 @@ def validate_profiles_jsons(json_data_dir: Path, json_schema_path: Path) -> int:
             logging.debug(f"Invalid profile file: {profiles_files_paths[i]}")
             continue
 
-        if _validate_profiles_json_data(json_data, schema_data):
+        if _validate_profiles_json_data(json_data, json_schema_data):
             result += 1
 
     return result
 
 
+def validate_profiles_jsons(json_data_dir: Path, json_schema_path: Path) -> int:
+    schema_data = load_schema_json(json_schema_path)
+    if schema_data is None:
+        logging.error(f"Invalid profile file: {json_schema_path}")
+        return 0
+
+    return validate_profiles_jsons_data(json_data_dir, schema_data)
+
+
 def load_schema_json(input_file):
     with open(input_file, "r", encoding="utf-8") as file:
         schema_file_data = json.load(file)
@@ -135,7 +140,6 @@ def load_profiles_jsons(input_dir):
 
     return json_files_dict
 
-
 class OutputFormatType(Enum):
     PRETTY = 'pretty'
     FLATTEN = 'flatten'
@@ -169,5 +173,3 @@ def save_profiles_jsons(json_files_dict, output_dir, format: OutputFormatType):
                 file.write(flat_json)
             else:
                 json.dump(value, file, indent=4)
-
-

scripts/source/vulkan_object_utils.py

@@ -40,7 +40,7 @@
 
 # Create the simplified, cached public function
 @functools.lru_cache(maxsize=1)
-def initVulkanObject(alternative_xml: str = None, video: bool = False) -> VulkanObject:
+def initVulkanObject(target_api: str = 'vulkan', alternative_xml: str = None, video: bool = False) -> VulkanObject:
     """
     Parses the bundled Vulkan registry (vk.xml) and returns the populated
     VulkanObject.
@@ -74,7 +74,7 @@ def generate(self):
         SetOutputDirectory(output_dir)
         SetOutputFileName("unused.txt")
         # TODO - Make a get_vulkan_sc_object() or pass this in as a parameter
-        SetTargetApiName('vulkan')
+        SetTargetApiName(target_api)
         SetMergedApiNames(None)
 
         xml_path = None

scripts/tests/test_init.py

@@ -35,7 +35,7 @@ class TestVulkanObjectInit(unittest.TestCase):
     registry_path = None
 
     def test_load_vulkan_object(self):
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

scripts/tests/test_vulkan_object_utils.py

@@ -36,7 +36,7 @@ class TestVulkanObjectUtils(unittest.TestCase):
     registry_path = None
 
     # def testVulkanObjectVersion(self):
-    #     vk: VulkanObject = initVulkanObject(self.registry_path)
+    #     vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
     #     VK_VERSION = buildVulkanVersionEnum(vk)
 
@@ -48,7 +48,7 @@ class TestVulkanObjectUtils(unittest.TestCase):
 
     # Check we can get the list of feature aliases from any feature structure
     def testVulkanObjectUtilsStructFeatureAliasesAccess(self):
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
         # Case 2: Building the list of aliases of an actual struct using the getAliases helper function that hide that not all structs are stored in vk.structs
         query_id2 = StructCapabilityAlias("VkPhysicalDeviceShaderSubgroupRotateFeatures", "shaderSubgroupRotate")
@@ -122,7 +122,7 @@ def testVulkanObjectUtilsStructFeatureAliasesAccess(self):
 
     # Check we can get the list of property aliases from any property structure
     def testVulkanObjectUtilsStructPropertyAliasesAccess(self):
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
         # Case 1: Building the list of aliases of an actual struct using the getCapabilityAliases helper function that hide that not all structs are stored in vk.structs
         query_id1 = StructCapabilityAlias("VkPhysicalDeviceLineRasterizationProperties", "lineSubPixelPrecisionBits")
@@ -197,7 +197,7 @@ def testVulkanObjectUtilsStructPropertyAliasesAccess(self):
         assert member_C_aliases == []
 
     def testFindExtensionVersion(self):
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
         extension_version0 = findExtensionVersion(vk, "VK_KHR_dynamic_rendering")
         self.assertEqual(extension_version0, 1)
@@ -211,7 +211,7 @@ def testFindExtensionVersion(self):
     def testGatherDependentExtensions(self):
         self.maxDiff = 1024
 
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
         extensions_data = {
             "VK_KHR_dynamic_rendering": 1,
@@ -336,7 +336,7 @@ def testGatherDynamicStructs(self):
         Verifies that gatherDynamicStructs correctly builds an automated layout 
         of valid dynamic array properties directly from the parsed VulkanObject.
         """
-        vk: VulkanObject = initVulkanObject(self.registry_path)
+        vk: VulkanObject = initVulkanObject('vulkan', self.registry_path)
 
         # Programmatically discover all extensible dynamic array property containers
         dynamic_structs = gatherDynamicStructs(vk)