From 03722369ce246fff55b45edf1e7e7a9753772879 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Fri, 20 Dec 2024 00:31:04 +0000 Subject: [PATCH 1/2] Separate out argument parsing --- .../TensileCreateLib/ParseArguments.py | 149 +++++++++++++++ .../Tensile/TensileCreateLib/__init__.py | 23 +++ tensilelite/Tensile/TensileCreateLibrary.py | 171 ++++-------------- 3 files changed, 212 insertions(+), 131 deletions(-) create mode 100644 tensilelite/Tensile/TensileCreateLib/ParseArguments.py create mode 100644 tensilelite/Tensile/TensileCreateLib/__init__.py diff --git a/tensilelite/Tensile/TensileCreateLib/ParseArguments.py b/tensilelite/Tensile/TensileCreateLib/ParseArguments.py new file mode 100644 index 0000000000..79d1521907 --- /dev/null +++ b/tensilelite/Tensile/TensileCreateLib/ParseArguments.py @@ -0,0 +1,149 @@ +################################################################################ +# +# Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import os +import argparse +from typing import Any, Dict, List, Optional + +from ..Common import architectureMap +from ..Utilities.Toolchain import ToolchainDefaults + +def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: + """Parse command line arguments for TensileCreateLibrary. + + Args: + input: List of strings representing command line arguments used when + calling parseArguments prgrammatically e.g. in testing. + + Returns: + A dictionary containing the keys representing options and their values. + """ + def splitExtraParameters(par): + """ + Allows the --global-parameters option to specify any parameters from the command line. + """ + (key, value) = par.split("=") + value = eval(value) + return (key, value) + + argParser = argparse.ArgumentParser() + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument("OutputPath", help="Where to write library files?") + argParser.add_argument("RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"]) + argParser.add_argument("--cxx-compiler", dest="CxxCompiler", action="store", default=ToolchainDefaults.CXX_COMPILER, + help=f"Default: {ToolchainDefaults.CXX_COMPILER}") + argParser.add_argument("--c-compiler", dest="CCompiler", action="store", default=ToolchainDefaults.C_COMPILER) + argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") + argParser.add_argument("--offload-bundler", dest="OffloadBundler", action="store", default=ToolchainDefaults.OFFLOAD_BUNDLER) + argParser.add_argument("--assembler", dest="Assembler", action="store", default=ToolchainDefaults.ASSEMBLER) + argParser.add_argument("--code-object-version", dest="CodeObjectVersion", choices=["default", "V4", "V5"], action="store") + argParser.add_argument("--architecture", dest="Architecture", type=str, action="store", default="all", help="Supported archs: " + " ".join(architectureMap.keys())) + argParser.add_argument("--short-file-names", dest="ShortNames", action="store_true") + argParser.add_argument("--no-short-file-names", dest="ShortNames", action="store_false") + argParser.add_argument("--library-print-debug", dest="LibraryPrintDebug", action="store_true") + argParser.add_argument("--no-library-print-debug", dest="LibraryPrintDebug", action="store_false") + argParser.add_argument("--no-compress", dest="NoCompress", action="store_true", help="Don't compress assembly code objects.") + argParser.add_argument("--experimental", dest="Experimental", action="store_true", + help="Include logic files in directories named 'Experimental'.") + argParser.add_argument("--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator.") + argParser.add_argument("--version", help="Version string to embed into library file.") + argParser.add_argument("--logic-format", dest="LogicFormat", choices=["yaml", "json"], \ + action="store", default="yaml", help="select which logic format to use") + argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], + action="store", default="msgpack", help="select which library format to use") + argParser.add_argument("--generate-sources-and-exit", dest="GenerateSourcesAndExit", action="store_true", + default=False, help="Output source files only and exit.") + argParser.add_argument("--jobs", "-j", dest="CpuThreads", type=int, + default=-1, help="Number of parallel jobs to launch.") + argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, + default=1, help="Set printout verbosity level.") + argParser.add_argument("--print-timing", dest="PrintTiming", + default=False, action="store_true", help="Print duration of each stage.") + argParser.add_argument("--separate-architectures", dest="SeparateArchitectures", action="store_true", + default=False, help="Separates TensileLibrary file by architecture") + argParser.add_argument("--lazy-library-loading", dest="LazyLibraryLoading", action="store_true", + default=False, help="Loads Tensile libraries when needed instead of upfront.") + argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", + default=False, help="Enable marker in Tensile.") + argParser.add_argument("--global-parameters", nargs="+", type=splitExtraParameters, default=[]) + argParser.add_argument("--no-generate-solution-table", dest="GenSolTable", action="store_false", default=True, + help="Skip generating solution-yaml matching table") + argParser.add_argument("--asm-debug", dest="AsmDebug", action="store_true", default=False, + help="Keep debug information for built code objects") + argParser.add_argument("--build-id", dest="BuildIdKind", action="store", default="sha1") + argParser.add_argument("--address-sanitizer", dest="AsanBuild", action="store_true", + default=False, help="Enable ASAN build.") + argParser.add_argument("--keep-build-tmp", dest="KeepBuildTmp", action="store_true", + default=False, help="Do not remove the temporary build directory (may required hundreds of GBs of space)"), + argParser.add_argument("--validate-library", dest="ValidateLibrary", action="store_true", default=False) + argParser.add_argument("--logic-filter", dest="LogicFilter", action="store", default="*", type=str, + help="Cutomsized logic filter, default is *, i.e. all logics." + " Example: gfx942/Equality/* for building equality of gfx942 only") + + args = argParser.parse_args() + + arguments = {} + arguments["RuntimeLanguage"] = args.RuntimeLanguage + arguments["CodeObjectVersion"] = args.CodeObjectVersion + arguments["Architecture"] = args.Architecture + arguments["SeparateArchitectures"] = args.SeparateArchitectures + arguments["LazyLibraryLoading"] = args.LazyLibraryLoading + arguments["EnableMarker"] = args.EnableMarker + if args.CmakeCxxCompiler: + os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler + arguments["ShortNames"] = args.ShortNames + arguments["LibraryPrintDebug"] = args.LibraryPrintDebug + arguments["CodeFromFiles"] = False + arguments["LogicFormat"] = args.LogicFormat + arguments["LibraryFormat"] = args.LibraryFormat + if args.no_enumerate: + arguments["AMDGPUArchPath"] = False + arguments["CpuThreads"] = args.CpuThreads + arguments["PrintLevel"] = args.PrintLevel + arguments["PrintTiming"] = args.PrintTiming + arguments["AsmDebug"] = args.AsmDebug + arguments["BuildIdKind"] = args.BuildIdKind + arguments["KeepBuildTmp"] = args.KeepBuildTmp + arguments["AsanBuild"] = args.AsanBuild + arguments["ValidateLibrary"] = args.ValidateLibrary + arguments["LogicPath"] = args.LogicPath + arguments["OutputPath"] = args.OutputPath + arguments["CxxCompiler"] = args.CxxCompiler + arguments["OffloadBundler"] = args.OffloadBundler + arguments["Assembler"] = args.Assembler + arguments["LibraryFormat"] = args.LibraryFormat + arguments["UseCompression"] = not args.NoCompress + arguments["LogicFilter"] = args.LogicFilter + arguments["Experimental"] = args.Experimental + arguments["GenSolTable"] = args.GenSolTable + arguments["CxxCompiler"] = args.CxxCompiler + arguments["CCompiler"] = args.CCompiler + arguments["OffloadBundler"] = args.OffloadBundler + arguments["Assembler"] = args.Assembler + arguments["Version"] = args.version + + for key, value in args.global_parameters: + arguments[key] = value + + return arguments diff --git a/tensilelite/Tensile/TensileCreateLib/__init__.py b/tensilelite/Tensile/TensileCreateLib/__init__.py new file mode 100644 index 0000000000..388e51fd32 --- /dev/null +++ b/tensilelite/Tensile/TensileCreateLib/__init__.py @@ -0,0 +1,23 @@ +################################################################################ +# +# Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ diff --git a/tensilelite/Tensile/TensileCreateLibrary.py b/tensilelite/Tensile/TensileCreateLibrary.py index 13fa40364c..6ce2229759 100644 --- a/tensilelite/Tensile/TensileCreateLibrary.py +++ b/tensilelite/Tensile/TensileCreateLibrary.py @@ -42,11 +42,11 @@ from .SolutionLibrary import MasterSolutionLibrary from .SolutionStructs import Solution from .CustomYamlLoader import load_logic_gfx_arch +from .TensileCreateLib.ParseArguments import parseArguments from .Utilities.Profile import profile from .Utilities.Toolchain import getVersion, validateToolchain, ToolchainDefaults from .BuildCommands import SourceCommands, AssemblyCommands -import argparse import collections import glob import itertools @@ -358,10 +358,10 @@ def generateKernelObjectsFromSolutions(solutions): def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): # skip the logic which architectureName is not in the build target. - if ";" in args.Architecture: - archs = args.Architecture.split(";") # user arg list format + if ";" in args["Architecture"]: + archs = args["Architecture"].split(";") # user arg list format else: - archs = args.Architecture.split("_") # workaround for cmake list in list issue + archs = args["Architecture"].split("_") # workaround for cmake list in list issue solutions = [] masterLibraries = {} @@ -389,15 +389,15 @@ def libraryIter(lib: MasterSolutionLibrary): nextSolIndex = masterLibraries[architectureName].merge(newLibrary, nextSolIndex) else: masterLibraries[architectureName] = newLibrary - masterLibraries[architectureName].version = args.version + masterLibraries[architectureName].version = args["Version"] else: if fullMasterLibrary is None: fullMasterLibrary = newLibrary - fullMasterLibrary.version = args.version + fullMasterLibrary.version = args["Version"] else: fullMasterLibrary.merge(newLibrary) - if args.GenSolTable: + if args["GenSolTable"]: # Match yaml file solutions to solution index for localIdx, _, s in libraryIter(newLibrary): matchTable[s.index] = [srcFile, localIdx] @@ -423,7 +423,7 @@ def libraryIter(lib: MasterSolutionLibrary): # remove duplicates while preserving order solutions = dict.fromkeys(solutions).keys() - if args.GenSolTable: + if args["GenSolTable"]: LibraryIO.write("MatchTable", matchTable) return solutions, masterLibraries, fullMasterLibrary @@ -458,121 +458,30 @@ def TensileCreateLibrary(): print2(HR) print2("") - ############################################################################## - # Parse Command Line Arguments - ############################################################################## - def splitExtraParameters(par): - """ - Allows the --global-parameters option to specify any parameters from the command line. - """ - - (key, value) = par.split("=") - value = eval(value) - return (key, value) - print2("Arguments: %s" % sys.argv) - argParser = argparse.ArgumentParser() - argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") - argParser.add_argument("OutputPath", help="Where to write library files?") - argParser.add_argument("RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"]) - argParser.add_argument("--cxx-compiler", dest="CxxCompiler", action="store", default=ToolchainDefaults.CXX_COMPILER, - help=f"Default: {ToolchainDefaults.CXX_COMPILER}") - argParser.add_argument("--c-compiler", dest="CCompiler", action="store", default=ToolchainDefaults.C_COMPILER) - argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") - argParser.add_argument("--offload-bundler", dest="OffloadBundler", action="store", default=ToolchainDefaults.OFFLOAD_BUNDLER) - argParser.add_argument("--assembler", dest="Assembler", action="store", default=ToolchainDefaults.ASSEMBLER) - argParser.add_argument("--code-object-version", dest="CodeObjectVersion", choices=["default", "V4", "V5"], action="store") - argParser.add_argument("--architecture", dest="Architecture", type=str, action="store", default="all", help="Supported archs: " + " ".join(architectureMap.keys())) - argParser.add_argument("--short-file-names", dest="ShortNames", action="store_true") - argParser.add_argument("--no-short-file-names", dest="ShortNames", action="store_false") - argParser.add_argument("--library-print-debug", dest="LibraryPrintDebug", action="store_true") - argParser.add_argument("--no-library-print-debug", dest="LibraryPrintDebug", action="store_false") - argParser.add_argument("--no-compress", dest="NoCompress", action="store_true", help="Don't compress assembly code objects.") - argParser.add_argument("--experimental", dest="Experimental", action="store_true", - help="Include logic files in directories named 'Experimental'.") - argParser.add_argument("--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator.") - argParser.add_argument("--version", help="Version string to embed into library file.") - argParser.add_argument("--logic-format", dest="LogicFormat", choices=["yaml", "json"], \ - action="store", default="yaml", help="select which logic format to use") - argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], - action="store", default="msgpack", help="select which library format to use") - argParser.add_argument("--generate-sources-and-exit", dest="GenerateSourcesAndExit", action="store_true", - default=False, help="Output source files only and exit.") - argParser.add_argument("--jobs", "-j", dest="CpuThreads", type=int, - default=-1, help="Number of parallel jobs to launch.") - argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, - default=1, help="Set printout verbosity level.") - argParser.add_argument("--print-timing", dest="PrintTiming", - default=False, action="store_true", help="Print duration of each stage.") - argParser.add_argument("--separate-architectures", dest="SeparateArchitectures", action="store_true", - default=False, help="Separates TensileLibrary file by architecture") - argParser.add_argument("--lazy-library-loading", dest="LazyLibraryLoading", action="store_true", - default=False, help="Loads Tensile libraries when needed instead of upfront.") - argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", - default=False, help="Enable marker in Tensile.") - argParser.add_argument("--global-parameters", nargs="+", type=splitExtraParameters, default=[]) - argParser.add_argument("--no-generate-solution-table", dest="GenSolTable", action="store_false", default=True, - help="Skip generating solution-yaml matching table") - argParser.add_argument("--asm-debug", dest="AsmDebug", action="store_true", default=False, - help="Keep debug information for built code objects") - argParser.add_argument("--build-id", dest="BuildIdKind", action="store", default="sha1") - argParser.add_argument("--address-sanitizer", dest="AsanBuild", action="store_true", - default=False, help="Enable ASAN build.") - argParser.add_argument("--keep-build-tmp", dest="KeepBuildTmp", action="store_true", - default=False, help="Do not remove the temporary build directory (may required hundreds of GBs of space)"), - argParser.add_argument("--validate-library", dest="ValidateLibrary", action="store_true", default=False) - argParser.add_argument("--logic-filter", dest="LogicFilter", action="store", default="*", type=str, - help="Cutomsized logic filter, default is *, i.e. all logics." - " Example: gfx942/Equality/* for building equality of gfx942 only") - - args = argParser.parse_args() - - logicPath = args.LogicPath - outputPath = args.OutputPath - cxxCompiler = args.CxxCompiler - offloadBundler = args.OffloadBundler - assembler = args.Assembler - libraryFormat = args.LibraryFormat - useCompression = not args.NoCompress + + arguments = parseArguments() + + logicPath = arguments["LogicPath"] + libraryFormat = arguments["LibraryFormat"] + logicFormat = arguments["LogicFormat"] + logicFilter = arguments["LogicFilter"] + experimental = arguments["Experimental"] + architecture = arguments["Architecture"] + cxxCompiler = arguments["CxxCompiler"] + offloadBundler = arguments["OffloadBundler"] + assembler = arguments["Assembler"] + useCompression = arguments["UseCompression"] + outputPath = arguments["OutputPath"] + ensurePath(outputPath) + outputPath = os.path.abspath(outputPath) + print2("OutputPath: %s" % outputPath) ensurePath(outputPath) outputPath = os.path.abspath(outputPath) - arguments = {} - arguments["RuntimeLanguage"] = args.RuntimeLanguage - arguments["CodeObjectVersion"] = args.CodeObjectVersion - arguments["Architecture"] = args.Architecture - arguments["SeparateArchitectures"] = args.SeparateArchitectures - arguments["LazyLibraryLoading"] = args.LazyLibraryLoading - arguments["EnableMarker"] = args.EnableMarker - if args.CmakeCxxCompiler: - os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler - arguments["ShortNames"] = args.ShortNames - arguments["LibraryPrintDebug"] = args.LibraryPrintDebug - arguments["CodeFromFiles"] = False - arguments["LogicFormat"] = args.LogicFormat - arguments["LibraryFormat"] = args.LibraryFormat - if args.no_enumerate: - arguments["AMDGPUArchPath"] = False - - arguments["GenerateSourcesAndExit"] = args.GenerateSourcesAndExit - if arguments["GenerateSourcesAndExit"]: - # Generated sources are preserved and go into output dir - arguments["WorkingPath"] = outputPath - - arguments["CpuThreads"] = args.CpuThreads - arguments["PrintLevel"] = args.PrintLevel - arguments["PrintTiming"] = args.PrintTiming - arguments["AsmDebug"] = args.AsmDebug - arguments["BuildIdKind"] = args.BuildIdKind - arguments["KeepBuildTmp"] = args.KeepBuildTmp - arguments["AsanBuild"] = args.AsanBuild - arguments["ValidateLibrary"] = args.ValidateLibrary - - for key, value in args.global_parameters: - arguments[key] = value cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( - args.CxxCompiler, args.CCompiler, args.OffloadBundler, args.Assembler, ToolchainDefaults.HIP_CONFIG + arguments["CxxCompiler"], arguments["CCompiler"], arguments["OffloadBundler"], arguments["Assembler"], ToolchainDefaults.HIP_CONFIG ) print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") @@ -585,14 +494,14 @@ def splitExtraParameters(par): arguments["AMDClangVersion"] = getVersion(cxxCompiler) assignGlobalParameters(arguments, cxxCompiler) - + if not os.path.exists(logicPath): printExit("LogicPath %s doesn't exist" % logicPath) - if ";" in arguments["Architecture"]: - archs = arguments["Architecture"].split(";") # user arg list format + if ";" in architecture: + archs = architecture.split(";") # user arg list format else: - archs = arguments["Architecture"].split("_") # workaround for cmake list in list issue + archs = architecture.split("_") # workaround for cmake list in list issue logicArchs = set() for arch in archs: if arch in architectureMap: @@ -602,12 +511,12 @@ def splitExtraParameters(par): # Recursive directory search logicExtFormat = ".yaml" - if args.LogicFormat == "yaml": + if logicFormat == "yaml": pass - elif args.LogicFormat == "json": + elif logicFormat == "json": logicExtFormat = ".json" else: - printExit("Unrecognized LogicFormat", args.LogicFormat) + printExit(f"Unrecognized LogicFormat {logicFormat}") def archMatch(arch: str, archs: List[str]): return (arch in archs) or any(a.startswith(arch) for a in archs) @@ -615,13 +524,13 @@ def archMatch(arch: str, archs: List[str]): def validLogicFile(p: Path): return p.suffix == logicExtFormat and ("all" in archs or archMatch(load_logic_gfx_arch(p), archs)) - globPattern = os.path.join(logicPath, f"**/{args.LogicFilter}{logicExtFormat}") + globPattern = os.path.join(logicPath, f"**/{logicFilter}{logicExtFormat}") print1(f"# LogicFilter: {globPattern}") logicFiles = (os.path.join(logicPath, file) for file in glob.iglob(globPattern, recursive=True)) logicFiles = [file for file in logicFiles if validLogicFile(Path(file))] - print1(f"# Experimental: {args.Experimental}") - if not args.Experimental: + print1(f"# Experimental: {experimental}") + if not experimental: logicFiles = [file for file in logicFiles if "experimental" not in map(str.lower, Path(file).parts)] print2(f"# LibraryLogicFiles: {len(logicFiles)}") @@ -634,7 +543,7 @@ def validLogicFile(p: Path): ############################################################################## # Parse logicData, solutions, and masterLibraries from logic files - solutions, masterLibraries, fullMasterLibrary = generateLogicDataAndSolutions(logicFiles, args, cxxCompiler) + solutions, masterLibraries, fullMasterLibrary = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) @@ -667,19 +576,19 @@ def validLogicFile(p: Path): else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_"+archName) newMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, Utils.state(newMasterLibrary), args.LibraryFormat) + LibraryIO.write(masterFile, Utils.state(newMasterLibrary), libraryFormat) #Write placeholder libraries for name, lib in newMasterLibrary.lazyLibraries.items(): filename = os.path.join(newLibraryDir, name) lib.applyNaming(kernelMinNaming) #@TODO Check to see if kernelMinNaming is correct - LibraryIO.write(filename, Utils.state(lib), args.LibraryFormat) + LibraryIO.write(filename, Utils.state(lib), libraryFormat) else: masterFile = os.path.join(newLibraryDir, "TensileLibrary") fullMasterLibrary.applyNaming = timing(fullMasterLibrary.applyNaming) fullMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, Utils.state(fullMasterLibrary), args.LibraryFormat) + LibraryIO.write(masterFile, Utils.state(fullMasterLibrary), libraryFormat) theMasterLibrary = fullMasterLibrary if globalParameters["SeparateArchitectures"]: From 3bfbf43fd19d7856ef6b5be33bcb885cdda8db00 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Fri, 20 Dec 2024 00:38:53 +0000 Subject: [PATCH 2/2] Remove logic for unsupported build modes --- tensilelite/Tensile/TensileCreateLibrary.py | 181 +++++++++++++++----- 1 file changed, 136 insertions(+), 45 deletions(-) diff --git a/tensilelite/Tensile/TensileCreateLibrary.py b/tensilelite/Tensile/TensileCreateLibrary.py index 6ce2229759..2fa1b57b2a 100644 --- a/tensilelite/Tensile/TensileCreateLibrary.py +++ b/tensilelite/Tensile/TensileCreateLibrary.py @@ -35,28 +35,28 @@ from . import Utils from .TensileInstructions import getGfxName, TensileInstructions from .Common import globalParameters, HR, print1, print2, printExit, ensurePath, \ - CHeader, CMakeHeader, assignGlobalParameters, \ + CHeader, assignGlobalParameters, \ architectureMap, printWarning, \ - splitArchs + IsaVersion from .KernelWriterAssembly import KernelWriterAssembly +from .KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H from .SolutionLibrary import MasterSolutionLibrary from .SolutionStructs import Solution from .CustomYamlLoader import load_logic_gfx_arch -from .TensileCreateLib.ParseArguments import parseArguments from .Utilities.Profile import profile +from .TensileInstructions.Utils import getCOVFromParam from .Utilities.Toolchain import getVersion, validateToolchain, ToolchainDefaults from .BuildCommands import SourceCommands, AssemblyCommands - +from .TensileCreateLib.ParseArguments import parseArguments import collections import glob import itertools import os -import re import shutil import sys from timeit import default_timer as timer from pathlib import Path -from typing import Sequence, List, Union +from typing import Sequence, List, Union, NamedTuple, Optional def timing(func): def wrapper(*args, **kwargs): @@ -358,10 +358,10 @@ def generateKernelObjectsFromSolutions(solutions): def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): # skip the logic which architectureName is not in the build target. - if ";" in args["Architecture"]: - archs = args["Architecture"].split(";") # user arg list format + if ";" in args.Architecture: + archs = args.Architecture.split(";") # user arg list format else: - archs = args["Architecture"].split("_") # workaround for cmake list in list issue + archs = args.Architecture.split("_") # workaround for cmake list in list issue solutions = [] masterLibraries = {} @@ -389,15 +389,15 @@ def libraryIter(lib: MasterSolutionLibrary): nextSolIndex = masterLibraries[architectureName].merge(newLibrary, nextSolIndex) else: masterLibraries[architectureName] = newLibrary - masterLibraries[architectureName].version = args["Version"] + masterLibraries[architectureName].version = args.version else: if fullMasterLibrary is None: fullMasterLibrary = newLibrary - fullMasterLibrary.version = args["Version"] + fullMasterLibrary.version = args.version else: fullMasterLibrary.merge(newLibrary) - if args["GenSolTable"]: + if args.GenSolTable: # Match yaml file solutions to solution index for localIdx, _, s in libraryIter(newLibrary): matchTable[s.index] = [srcFile, localIdx] @@ -423,7 +423,7 @@ def libraryIter(lib: MasterSolutionLibrary): # remove duplicates while preserving order solutions = dict.fromkeys(solutions).keys() - if args["GenSolTable"]: + if args.GenSolTable: LibraryIO.write("MatchTable", matchTable) return solutions, masterLibraries, fullMasterLibrary @@ -458,30 +458,121 @@ def TensileCreateLibrary(): print2(HR) print2("") - print2("Arguments: %s" % sys.argv) - - arguments = parseArguments() - - logicPath = arguments["LogicPath"] - libraryFormat = arguments["LibraryFormat"] - logicFormat = arguments["LogicFormat"] - logicFilter = arguments["LogicFilter"] - experimental = arguments["Experimental"] - architecture = arguments["Architecture"] - cxxCompiler = arguments["CxxCompiler"] - offloadBundler = arguments["OffloadBundler"] - assembler = arguments["Assembler"] - useCompression = arguments["UseCompression"] - outputPath = arguments["OutputPath"] - ensurePath(outputPath) - outputPath = os.path.abspath(outputPath) + ############################################################################## + # Parse Command Line Arguments + ############################################################################## + def splitExtraParameters(par): + """ + Allows the --global-parameters option to specify any parameters from the command line. + """ + (key, value) = par.split("=") + value = eval(value) + return (key, value) + + print2("Arguments: %s" % sys.argv) + argParser = argparse.ArgumentParser() + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument("OutputPath", help="Where to write library files?") + argParser.add_argument("RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"]) + argParser.add_argument("--cxx-compiler", dest="CxxCompiler", action="store", default=ToolchainDefaults.CXX_COMPILER, + help=f"Default: {ToolchainDefaults.CXX_COMPILER}") + argParser.add_argument("--c-compiler", dest="CCompiler", action="store", default=ToolchainDefaults.C_COMPILER) + argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") + argParser.add_argument("--offload-bundler", dest="OffloadBundler", action="store", default=ToolchainDefaults.OFFLOAD_BUNDLER) + argParser.add_argument("--assembler", dest="Assembler", action="store", default=ToolchainDefaults.ASSEMBLER) + argParser.add_argument("--code-object-version", dest="CodeObjectVersion", choices=["default", "V4", "V5"], action="store") + argParser.add_argument("--architecture", dest="Architecture", type=str, action="store", default="all", help="Supported archs: " + " ".join(architectureMap.keys())) + argParser.add_argument("--short-file-names", dest="ShortNames", action="store_true") + argParser.add_argument("--no-short-file-names", dest="ShortNames", action="store_false") + argParser.add_argument("--library-print-debug", dest="LibraryPrintDebug", action="store_true") + argParser.add_argument("--no-library-print-debug", dest="LibraryPrintDebug", action="store_false") + argParser.add_argument("--no-compress", dest="NoCompress", action="store_true", help="Don't compress assembly code objects.") + argParser.add_argument("--experimental", dest="Experimental", action="store_true", + help="Include logic files in directories named 'Experimental'.") + argParser.add_argument("--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator.") + argParser.add_argument("--version", help="Version string to embed into library file.") + argParser.add_argument("--logic-format", dest="LogicFormat", choices=["yaml", "json"], \ + action="store", default="yaml", help="select which logic format to use") + argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], + action="store", default="msgpack", help="select which library format to use") + argParser.add_argument("--generate-sources-and-exit", dest="GenerateSourcesAndExit", action="store_true", + default=False, help="Output source files only and exit.") + argParser.add_argument("--jobs", "-j", dest="CpuThreads", type=int, + default=-1, help="Number of parallel jobs to launch.") + argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, + default=1, help="Set printout verbosity level.") + argParser.add_argument("--print-timing", dest="PrintTiming", + default=False, action="store_true", help="Print duration of each stage.") + argParser.add_argument("--separate-architectures", dest="SeparateArchitectures", action="store_true", + default=False, help="Separates TensileLibrary file by architecture") + argParser.add_argument("--lazy-library-loading", dest="LazyLibraryLoading", action="store_true", + default=False, help="Loads Tensile libraries when needed instead of upfront.") + argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", + default=False, help="Enable marker in Tensile.") + argParser.add_argument("--global-parameters", nargs="+", type=splitExtraParameters, default=[]) + argParser.add_argument("--no-generate-solution-table", dest="GenSolTable", action="store_false", default=True, + help="Skip generating solution-yaml matching table") + argParser.add_argument("--asm-debug", dest="AsmDebug", action="store_true", default=False, + help="Keep debug information for built code objects") + argParser.add_argument("--build-id", dest="BuildIdKind", action="store", default="sha1") + argParser.add_argument("--address-sanitizer", dest="AsanBuild", action="store_true", + default=False, help="Enable ASAN build.") + argParser.add_argument("--keep-build-tmp", dest="KeepBuildTmp", action="store_true", + default=False, help="Do not remove the temporary build directory (may required hundreds of GBs of space)"), + argParser.add_argument("--validate-library", dest="ValidateLibrary", action="store_true", default=False) + argParser.add_argument("--logic-filter", dest="LogicFilter", action="store", default="*", type=str, + help="Cutomsized logic filter, default is *, i.e. all logics." + " Example: gfx942/Equality/* for building equality of gfx942 only") + + args = argParser.parse_args() + + logicPath = args.LogicPath + outputPath = args.OutputPath + cxxCompiler = args.CxxCompiler + offloadBundler = args.OffloadBundler + assembler = args.Assembler + libraryFormat = args.LibraryFormat + useCompression = not args.NoCompress print2("OutputPath: %s" % outputPath) ensurePath(outputPath) outputPath = os.path.abspath(outputPath) + arguments = {} + arguments["RuntimeLanguage"] = args.RuntimeLanguage + arguments["CodeObjectVersion"] = args.CodeObjectVersion + arguments["Architecture"] = args.Architecture + arguments["SeparateArchitectures"] = args.SeparateArchitectures + arguments["LazyLibraryLoading"] = args.LazyLibraryLoading + arguments["EnableMarker"] = args.EnableMarker + if args.CmakeCxxCompiler: + os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler + arguments["ShortNames"] = args.ShortNames + arguments["LibraryPrintDebug"] = args.LibraryPrintDebug + arguments["CodeFromFiles"] = False + arguments["LogicFormat"] = args.LogicFormat + arguments["LibraryFormat"] = args.LibraryFormat + if args.no_enumerate: + arguments["AMDGPUArchPath"] = False + + arguments["GenerateSourcesAndExit"] = args.GenerateSourcesAndExit + if arguments["GenerateSourcesAndExit"]: + # Generated sources are preserved and go into output dir + arguments["WorkingPath"] = outputPath + + arguments["CpuThreads"] = args.CpuThreads + arguments["PrintLevel"] = args.PrintLevel + arguments["PrintTiming"] = args.PrintTiming + arguments["AsmDebug"] = args.AsmDebug + arguments["BuildIdKind"] = args.BuildIdKind + arguments["KeepBuildTmp"] = args.KeepBuildTmp + arguments["AsanBuild"] = args.AsanBuild + arguments["ValidateLibrary"] = args.ValidateLibrary + + for key, value in args.global_parameters: + arguments[key] = value cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( - arguments["CxxCompiler"], arguments["CCompiler"], arguments["OffloadBundler"], arguments["Assembler"], ToolchainDefaults.HIP_CONFIG + args.CxxCompiler, args.CCompiler, args.OffloadBundler, args.Assembler, ToolchainDefaults.HIP_CONFIG ) print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") @@ -494,14 +585,14 @@ def TensileCreateLibrary(): arguments["AMDClangVersion"] = getVersion(cxxCompiler) assignGlobalParameters(arguments, cxxCompiler) - + if not os.path.exists(logicPath): printExit("LogicPath %s doesn't exist" % logicPath) - if ";" in architecture: - archs = architecture.split(";") # user arg list format + if ";" in arguments["Architecture"]: + archs = arguments["Architecture"].split(";") # user arg list format else: - archs = architecture.split("_") # workaround for cmake list in list issue + archs = arguments["Architecture"].split("_") # workaround for cmake list in list issue logicArchs = set() for arch in archs: if arch in architectureMap: @@ -511,12 +602,12 @@ def TensileCreateLibrary(): # Recursive directory search logicExtFormat = ".yaml" - if logicFormat == "yaml": + if args.LogicFormat == "yaml": pass - elif logicFormat == "json": + elif args.LogicFormat == "json": logicExtFormat = ".json" else: - printExit(f"Unrecognized LogicFormat {logicFormat}") + printExit("Unrecognized LogicFormat", args.LogicFormat) def archMatch(arch: str, archs: List[str]): return (arch in archs) or any(a.startswith(arch) for a in archs) @@ -524,13 +615,13 @@ def archMatch(arch: str, archs: List[str]): def validLogicFile(p: Path): return p.suffix == logicExtFormat and ("all" in archs or archMatch(load_logic_gfx_arch(p), archs)) - globPattern = os.path.join(logicPath, f"**/{logicFilter}{logicExtFormat}") + globPattern = os.path.join(logicPath, f"**/{args.LogicFilter}{logicExtFormat}") print1(f"# LogicFilter: {globPattern}") logicFiles = (os.path.join(logicPath, file) for file in glob.iglob(globPattern, recursive=True)) logicFiles = [file for file in logicFiles if validLogicFile(Path(file))] - print1(f"# Experimental: {experimental}") - if not experimental: + print1(f"# Experimental: {args.Experimental}") + if not args.Experimental: logicFiles = [file for file in logicFiles if "experimental" not in map(str.lower, Path(file).parts)] print2(f"# LibraryLogicFiles: {len(logicFiles)}") @@ -543,7 +634,7 @@ def validLogicFile(p: Path): ############################################################################## # Parse logicData, solutions, and masterLibraries from logic files - solutions, masterLibraries, fullMasterLibrary = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) + solutions, masterLibraries, fullMasterLibrary = generateLogicDataAndSolutions(logicFiles, args, cxxCompiler) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) @@ -576,19 +667,19 @@ def validLogicFile(p: Path): else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_"+archName) newMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, Utils.state(newMasterLibrary), libraryFormat) + LibraryIO.write(masterFile, Utils.state(newMasterLibrary), args.LibraryFormat) #Write placeholder libraries for name, lib in newMasterLibrary.lazyLibraries.items(): filename = os.path.join(newLibraryDir, name) lib.applyNaming(kernelMinNaming) #@TODO Check to see if kernelMinNaming is correct - LibraryIO.write(filename, Utils.state(lib), libraryFormat) + LibraryIO.write(filename, Utils.state(lib), args.LibraryFormat) else: masterFile = os.path.join(newLibraryDir, "TensileLibrary") fullMasterLibrary.applyNaming = timing(fullMasterLibrary.applyNaming) fullMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, Utils.state(fullMasterLibrary), libraryFormat) + LibraryIO.write(masterFile, Utils.state(fullMasterLibrary), args.LibraryFormat) theMasterLibrary = fullMasterLibrary if globalParameters["SeparateArchitectures"]: